Skip to content

Commit 1cd4c51

Browse files
ahmedmuhsinahmedmuhsin
authored
[Preview] SDK types - BlobClient and BlobContainerClient (#813)
* initial commit * hook up model binding data and create one invocationChain factory per function * add parameter checking logic and an object cache * move sdktype lib * remove the need to use middlewarecontext in sdktype * move sdktypes lib to java-additions * update protobuf to a version that supports arm * use sdkmetadata instead of sdktypes directly * add shaded azure-identity in case user doesn't include it * pipeline workaround * add emulated tests for blobclient and blobcontainerclient * remove azure-identity from the worker since it causes conflicts even when shaded * fix emulated tests * put sdk types feature behind an app setting * fix build for linux testing * remove depends * update flag name * fix dependsOn * update build * update official build pipeline * change branch back to sdk-types * disable 17 and 21 temporarily due to a spotbugs plugin issue * update repo and branch
1 parent 701f380 commit 1cd4c51

File tree

25 files changed

+810
-82
lines changed

25 files changed

+810
-82
lines changed

‎emulatedtests/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E/Constants.cs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ public static class Constants
2424
public static string TriggerInputBindingBlobContainer = "test-triggerinput-java-new";
2525
public static string InputBindingBlobContainer = "test-input-java-new";
2626
public static string OutputBindingBlobContainer = "test-output-java-new";
27+
public static string TriggerInputBindingBlobClientSdk = "test-triggerinput-blobclient";
28+
public static string TriggerInputBindingBlobContainerClientSdk = "test-triggerinput-blobcontclient";
2729

2830
// Xunit Fixtures and Collections
2931
public const string FunctionAppCollectionName = "FunctionAppCollection";

‎emulatedtests/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E/Helpers/StorageHelpers.cs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,13 +84,17 @@ public async static Task ClearBlobContainers()
8484
await ClearBlobContainer(Constants.TriggerInputBindingBlobContainer);
8585
await ClearBlobContainer(Constants.InputBindingBlobContainer);
8686
await ClearBlobContainer(Constants.OutputBindingBlobContainer);
87+
await ClearBlobContainer(Constants.TriggerInputBindingBlobClientSdk);
88+
await ClearBlobContainer(Constants.TriggerInputBindingBlobContainerClientSdk);
8789
}
8890

8991
public async static Task CreateBlobContainers()
9092
{
9193
await CreateBlobContainer(Constants.TriggerInputBindingBlobContainer);
9294
await CreateBlobContainer(Constants.InputBindingBlobContainer);
9395
await CreateBlobContainer(Constants.OutputBindingBlobContainer);
96+
await CreateBlobContainer(Constants.TriggerInputBindingBlobClientSdk);
97+
await CreateBlobContainer(Constants.TriggerInputBindingBlobContainerClientSdk);
9498
}
9599

96100
public async static Task UpdloadFileToContainer(string containerName, string expectedFileName)

‎emulatedtests/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E/StorageEndToEndTests.cs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,5 +110,45 @@ public async Task BlobTriggerToBlob_Succeeds()
110110

111111
Assert.Equal("Hello World", result);
112112
}
113+
114+
[Fact]
115+
[Trait("Category", "SdkTypes")]
116+
public async Task BlobTriggerToBlob_BlobClient_Succeeds()
117+
{
118+
string fileName = Guid.NewGuid().ToString();
119+
120+
//cleanup
121+
await StorageHelpers.ClearBlobContainers();
122+
123+
//Setup
124+
await StorageHelpers.CreateBlobContainers();
125+
126+
//Trigger
127+
await StorageHelpers.UpdloadFileToContainer(Constants.TriggerInputBindingBlobClientSdk, fileName);
128+
129+
//Verify
130+
string result = await StorageHelpers.DownloadFileFromContainer(Constants.OutputBindingBlobContainer, "testfile");
131+
Assert.Equal("Hello World", result);
132+
}
133+
134+
[Fact]
135+
[Trait("Category", "SdkTypes")]
136+
public async Task BlobTriggerToBlob_BlobContainerClient_Succeeds()
137+
{
138+
string fileName = Guid.NewGuid().ToString();
139+
140+
//cleanup
141+
await StorageHelpers.ClearBlobContainers();
142+
143+
//Setup
144+
await StorageHelpers.CreateBlobContainers();
145+
146+
//Trigger
147+
await StorageHelpers.UpdloadFileToContainer(Constants.TriggerInputBindingBlobContainerClientSdk, fileName);
148+
149+
//Verify
150+
string result = await StorageHelpers.DownloadFileFromContainer(Constants.OutputBindingBlobContainer, "testfile");
151+
Assert.Equal("Hello World", result);
152+
}
113153
}
114154
}

‎emulatedtests/pom.xml

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,14 +21,16 @@
2121
<properties>
2222
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
2323
<java.version>1.8</java.version>
24-
<azure.functions.maven.plugin.version>1.18.0</azure.functions.maven.plugin.version>
24+
<azure.functions.maven.plugin.version>1.37.1</azure.functions.maven.plugin.version>
2525
<azure.functions.java.library.version>3.1.0</azure.functions.java.library.version>
2626
<azure.functions.java.library.sql.version>2.1.0</azure.functions.java.library.sql.version>
2727
<durabletask.azure.functions>1.0.0-beta.1</durabletask.azure.functions>
2828
<functionAppName>azure-functions-java-emulatedtests</functionAppName>
2929
<functionAppRegion>westus</functionAppRegion>
3030
<functionResourceGroup>java-functions-group</functionResourceGroup>
3131
<start-class>com.microsoft.azure.functions.endtoend.springcloud.Config</start-class>
32+
<!-- Empty by default: no exclusion -->
33+
<excludedClassPattern>**/___noop___.java</excludedClassPattern>
3234
</properties>
3335

3436
<repositories>
@@ -112,6 +114,11 @@
112114
<artifactId>spring-cloud-starter-function-web</artifactId>
113115
<scope>provided</scope>
114116
</dependency>
117+
<dependency>
118+
<groupId>com.azure</groupId>
119+
<artifactId>azure-storage-blob</artifactId>
120+
<version>12.29.0</version>
121+
</dependency>
115122
</dependencies>
116123

117124
<dependencyManagement>
@@ -136,6 +143,9 @@
136143
<source>${java.version}</source>
137144
<target>${java.version}</target>
138145
<encoding>${project.build.sourceEncoding}</encoding>
146+
<excludes>
147+
<exclude>${excludedClassPattern}</exclude>
148+
</excludes>
139149
</configuration>
140150
</plugin>
141151
<plugin>

‎emulatedtests/src/main/java/com/microsoft/azure/functions/endtoend/BlobTriggerSdkTypesTests.java

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
package com.microsoft.azure.functions.endtoend;
2+
3+
import com.azure.storage.blob.BlobClient;
4+
import com.azure.storage.blob.BlobContainerClient;
5+
import com.microsoft.azure.functions.ExecutionContext;
6+
import com.microsoft.azure.functions.OutputBinding;
7+
import com.microsoft.azure.functions.annotation.*;
8+
9+
import java.io.ByteArrayOutputStream;
10+
11+
/**
12+
* Azure Functions with Azure Storage Blob.
13+
*/
14+
public class BlobTriggerSdkTypesTests {
15+
/**
16+
* This function will be invoked when a new or updated blob is detected at the specified path. The blob contents are provided as input to this function.
17+
*/
18+
@FunctionName("BlobTriggerUsingBlobClientToBlobTest")
19+
@StorageAccount("AzureWebJobsStorage")
20+
public void BlobTriggerToBlobTest_BlobClient(
21+
@BlobTrigger(name = "triggerBlob", path = "test-triggerinput-blobclient/{name}", dataType = "binary") BlobClient triggerBlobClient,
22+
@BindingName("name") String fileName,
23+
@BlobOutput(name = "outputBlob", path = "test-output-java-new/testfile.txt", dataType = "binary") OutputBinding<byte[]> outputBlob,
24+
final ExecutionContext context
25+
) {
26+
context.getLogger().info("BlobTriggerUsingBlobClient triggered for blob: " + fileName);
27+
28+
// Download the blob content
29+
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
30+
triggerBlobClient.downloadStream(outputStream);
31+
32+
// Set the downloaded content as output
33+
outputBlob.setValue(outputStream.toByteArray());
34+
context.getLogger().info("Uploaded blob " + fileName + " to container test-output-java-new/testfile.txt");
35+
}
36+
37+
/**
38+
* This function will be invoked when a new or updated blob is detected at the specified path. The blob contents are provided as input to this function.
39+
*/
40+
@FunctionName("BlobTriggerUsingBlobContainerClientToBlobTest")
41+
@StorageAccount("AzureWebJobsStorage")
42+
public void BlobTriggerToBlobTest_BlobContainerClient(
43+
@BlobTrigger(name = "triggerBlob", path = "test-triggerinput-blobcontclient/{name}", dataType = "binary") BlobContainerClient triggerBlobContainerClient,
44+
@BindingName("name") String fileName,
45+
@BlobOutput(name = "outputBlob", path = "test-output-java-new/testfile.txt", dataType = "binary") OutputBinding<byte[]> outputBlob,
46+
final ExecutionContext context
47+
) {
48+
context.getLogger().info("BlobTriggerUsingBlobContainerClient triggered for blob: " + fileName);
49+
50+
// Download the blob content
51+
BlobClient triggerBlobClient = triggerBlobContainerClient.getBlobClient(fileName);
52+
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
53+
triggerBlobClient.downloadStream(outputStream);
54+
55+
// Set the downloaded content as output
56+
outputBlob.setValue(outputStream.toByteArray());
57+
context.getLogger().info("Uploaded blob " + fileName + " to container test-output-java-new/testfile.txt");
58+
}
59+
}

‎emulatedtests/src/main/java/com/microsoft/azure/functions/endtoend/BlobTriggerTests.java

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22

33
import com.microsoft.azure.functions.annotation.*;
44

5-
import java.util.List;
6-
import java.util.Optional;
7-
85
import com.microsoft.azure.functions.*;
96

107
/**
@@ -56,7 +53,7 @@ public void BlobTriggerStringTest(
5653
context.getLogger().info("Java Blob trigger function BlobTriggerStringTest processed a blob.\n Name: " + fileName + "\n Content: " + triggerBlobText);
5754
outputBlob.setValue(triggerBlobText);
5855
}
59-
56+
6057
public static class TestBlobData {
6158
public String blobText;
6259
}

‎eng/ci/public-build.yml

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@ resources:
2424
name: 1ESPipelineTemplates/1ESPipelineTemplates
2525
ref: refs/tags/release
2626

27+
parameters:
28+
- name: runSdkTypesTests
29+
type: boolean
30+
default: false
31+
2732
extends:
2833
template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
2934
parameters:
@@ -50,15 +55,17 @@ extends:
5055
- template: /eng/ci/templates/jobs/build.yml@self
5156

5257
- stage: TestWindows
53-
dependsOn: Build
58+
dependsOn: []
5459
jobs:
5560
- template: /eng/ci/templates/jobs/run-emulated-tests-windows.yml@self
5661
parameters:
5762
poolName: 1es-pool-azfunc-public
63+
runSdkTypesTests: ${{ parameters.runSdkTypesTests }}
5864

5965
- stage: TestLinux
60-
dependsOn: Build
66+
dependsOn: []
6167
jobs:
6268
- template: /eng/ci/templates/jobs/run-emulated-tests-linux.yml@self
6369
parameters:
6470
poolName: 1es-pool-azfunc-public
71+
runSdkTypesTests: ${{ parameters.runSdkTypesTests }}

‎eng/ci/templates/jobs/build.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@ jobs:
1414
- pwsh: |
1515
java -version
1616
displayName: 'Check default java version'
17+
- pwsh: |
18+
.\installAdditionsLocally.ps1
19+
displayName: 'Install java-additions locally'
1720
- pwsh: |
1821
mvn clean package
1922
displayName: 'Build java worker'

‎eng/ci/templates/jobs/run-emulated-tests-linux.yml

Lines changed: 87 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
parameters:
2-
poolName: ''
2+
- name: poolName
3+
type: string
4+
default: ''
5+
- name: runSdkTypesTests
6+
type: boolean
7+
default: false
38

49
jobs:
510
- job: "TestLinux"
@@ -29,16 +34,16 @@ jobs:
2934
JAVA_VERSION: 'microsoft-jdk-11.0.21-linux-x64'
3035
JDK_PATH: 'jdk-11.0.21+9'
3136
JAVA_VERSION_SPEC: '11'
32-
microsoft-open-jdk-17-linux:
33-
JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-17.0.9-linux-x64.tar.gz'
34-
JAVA_VERSION: 'microsoft-jdk-17.0.9-linux-x64'
35-
JDK_PATH: 'jdk-17.0.9+8'
36-
JAVA_VERSION_SPEC: '17'
37-
microsoft-open-jdk-21-linux:
38-
JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-21.0.1-linux-x64.tar.gz'
39-
JAVA_VERSION: 'microsoft-jdk-21.0.1-linux-x64'
40-
JDK_PATH: 'jdk-21.0.1+12'
41-
JAVA_VERSION_SPEC: '21'
37+
# microsoft-open-jdk-17-linux:
38+
# JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-17.0.9-linux-x64.tar.gz'
39+
# JAVA_VERSION: 'microsoft-jdk-17.0.9-linux-x64'
40+
# JDK_PATH: 'jdk-17.0.9+8'
41+
# JAVA_VERSION_SPEC: '17'
42+
# microsoft-open-jdk-21-linux:
43+
# JDK_DOWNLOAD_LINK: 'https://aka.ms/download-jdk/microsoft-jdk-21.0.1-linux-x64.tar.gz'
44+
# JAVA_VERSION: 'microsoft-jdk-21.0.1-linux-x64'
45+
# JDK_PATH: 'jdk-21.0.1+12'
46+
# JAVA_VERSION_SPEC: '21'
4247

4348
steps:
4449
- task: NuGetToolInstaller@1
@@ -57,18 +62,6 @@ jobs:
5762
- pwsh: |
5863
java -version
5964
displayName: 'Check default java version'
60-
- pwsh: |
61-
if ("$(isTag)"){
62-
$buildNumber="$(Build.SourceBranchName)"
63-
Write-Host "Found git tag."
64-
}
65-
else {
66-
$buildNumber="$(Build.BuildNumber)-v4"
67-
Write-Host "git tag not found. Setting package suffix to '$buildNumber'"
68-
}
69-
Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber"
70-
.\package-pipeline.ps1 -buildNumber $buildNumber
71-
displayName: 'Executing build script'
7265
- task: UseDotNet@2
7366
displayName: 'Install .NET 6'
7467
inputs:
@@ -87,6 +80,30 @@ jobs:
8780
jdkDestinationDirectory: "$(downloadPath)/externals"
8881
cleanDestinationDirectory: true
8982
displayName: 'Setup Java for Linux'
83+
- bash: |
84+
docker compose -f emulatedtests/utils/docker-compose.yml pull
85+
docker compose -f emulatedtests/utils/docker-compose.yml up -d
86+
displayName: 'Install Azurite and Start Emulators'
87+
- pwsh: |
88+
.\installAdditionsLocally.ps1
89+
displayName: 'Install java-additions locally'
90+
- pwsh: |
91+
if ("$(isTag)"){
92+
$buildNumber="$(Build.SourceBranchName)"
93+
Write-Host "Found git tag."
94+
}
95+
else {
96+
$buildNumber="$(Build.BuildNumber)-v4"
97+
Write-Host "git tag not found. Setting package suffix to '$buildNumber'"
98+
}
99+
Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber"
100+
.\package-pipeline.ps1 -buildNumber $buildNumber
101+
displayName: 'Executing build script'
102+
- pwsh: |
103+
cd ./emulatedtests
104+
mvn clean package -DexcludedClassPattern="**/BlobTriggerSdkTypesTests.java" `-Dmaven`.javadoc`.skip=true `-Dmaven`.test`.skip `-Dorg`.slf4j`.simpleLogger`.log`.org`.apache`.maven`.cli`.transfer`.Slf4jMavenTransferListener=warn `-B
105+
Copy-Item "confluent_cloud_cacert.pem" "./target/azure-functions/azure-functions-java-emulatedtests"
106+
displayName: 'Package Java for E2E'
90107
- pwsh: |
91108
.\setup-tests-pipeline.ps1
92109
displayName: 'Setup test environment -- Install the Core Tools'
@@ -96,23 +113,60 @@ jobs:
96113
export PATH=$PATH:./Azure.Functions.Cli
97114
func --version
98115
displayName: 'Setup Core Tools - Linux'
99-
- pwsh: |
100-
cd ./emulatedtests
101-
mvn clean package `-Dmaven`.javadoc`.skip=true `-Dmaven`.test`.skip `-Dorg`.slf4j`.simpleLogger`.log`.org`.apache`.maven`.cli`.transfer`.Slf4jMavenTransferListener=warn `-B
102-
Copy-Item "confluent_cloud_cacert.pem" "./target/azure-functions/azure-functions-java-emulatedtests"
103-
displayName: 'Package Java for E2E'
104-
- bash: |
105-
docker compose -f emulatedtests/utils/docker-compose.yml pull
106-
docker compose -f emulatedtests/utils/docker-compose.yml up -d
107-
displayName: 'Install Azurite and Start Emulators'
108116
- task: DotNetCoreCLI@2
109117
retryCountOnTaskFailure: 3
110118
inputs:
111119
command: 'test'
112120
projects: |
113121
emulatedtests/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E.csproj
122+
arguments: '--filter "Category!=SdkTypes"'
114123
env:
115124
JAVA_HOME: $(JavaHome)
116125
AzureWebJobsStorage: "UseDevelopmentStorage=true"
117126
displayName: 'Build & Run tests'
118-
continueOnError: false
127+
continueOnError: false
128+
# ------------------------------------------
129+
# Conditionally run an additional set of steps for "SDK types" scenario
130+
# ------------------------------------------
131+
- ${{ if eq(parameters.runSdkTypesTests, true) }}:
132+
- pwsh: |
133+
.\installMavenPluginLocally.ps1
134+
displayName: 'Install maven plugin locally'
135+
- pwsh: |
136+
if ("$(isTag)"){
137+
$buildNumber="$(Build.SourceBranchName)"
138+
Write-Host "Found git tag."
139+
}
140+
else {
141+
$buildNumber="$(Build.BuildNumber)-v4"
142+
Write-Host "git tag not found. Setting package suffix to '$buildNumber'"
143+
}
144+
Write-Host "##vso[task.setvariable variable=buildNumber;isOutput=true;]$buildNumber"
145+
.\package-pipeline.ps1 -buildNumber $buildNumber
146+
displayName: 'Executing build script'
147+
- pwsh: |
148+
cd ./emulatedtests
149+
mvn clean package `-Dmaven`.javadoc`.skip=true `-Dmaven`.test`.skip `-Dorg`.slf4j`.simpleLogger`.log`.org`.apache`.maven`.cli`.transfer`.Slf4jMavenTransferListener=warn `-B
150+
Copy-Item "confluent_cloud_cacert.pem" "./target/azure-functions/azure-functions-java-emulatedtests"
151+
displayName: 'Package Java for E2E'
152+
- pwsh: |
153+
.\setup-tests-pipeline.ps1
154+
displayName: 'Setup test environment -- Install the Core Tools'
155+
- bash: |
156+
chmod +x ./Azure.Functions.Cli/func
157+
chmod +x ./Azure.Functions.Cli/gozip
158+
export PATH=$PATH:./Azure.Functions.Cli
159+
func --version
160+
displayName: 'Setup Core Tools - Linux'
161+
- task: DotNetCoreCLI@2
162+
retryCountOnTaskFailure: 3
163+
inputs:
164+
command: 'test'
165+
projects: |
166+
emulatedtests/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E/Azure.Functions.Java.Tests.E2E.csproj
167+
env:
168+
JAVA_HOME: $(JavaHome)
169+
AzureWebJobsStorage: "UseDevelopmentStorage=true"
170+
ENABLE_SDK_TYPES: "true"
171+
displayName: 'Build & Run tests'
172+
continueOnError: false

0 commit comments

Comments
 (0)