Skip to content
This repository was archived by the owner on Nov 1, 2023. It is now read-only.

Commit 0141320

Browse files
authored
Merge branch 'main' into dependabot/nuget/src/ApiService/Azure.Identity-1.8.2
2 parents 18783af + 4084d0a commit 0141320

File tree

194 files changed

+2242
-633
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

194 files changed

+2242
-633
lines changed

.github/workflows/ci.yml

Lines changed: 15 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ env:
2020
SCCACHE_CACHE_SIZE: 1G
2121
ACTIONS_CACHE_KEY_DATE: 2022-11-21-02
2222
CI: true
23-
DOTNET_VERSION: 7.0.x
2423

2524
jobs:
2625
agent:
@@ -37,6 +36,11 @@ jobs:
3736
"1ES.Pool=onefuzz-ci",
3837
"1ES.ImageOverride=github-runner-image-ubuntu-20.04",
3938
]
39+
- [
40+
self-hosted,
41+
"1ES.Pool=onefuzz-ci-arm64",
42+
"1ES.ImageOverride=github-runner-image-ubuntu-20.04-arm64",
43+
]
4044
runs-on: "${{ matrix.os }}"
4145
steps:
4246
- uses: actions/checkout@v3
@@ -269,8 +273,8 @@ jobs:
269273
- name: Setup .NET Core SDK
270274
uses: actions/setup-dotnet@v3
271275
with:
272-
dotnet-version: ${{env.DOTNET_VERSION}}
273-
dotnet-quality: "ga"
276+
# use global.json to install the correct version
277+
global-json-file: global.json
274278
- name: Install dependencies
275279
run: |
276280
cd src/ApiService/
@@ -364,8 +368,8 @@ jobs:
364368
- name: Setup .NET Core SDK
365369
uses: actions/setup-dotnet@v3
366370
with:
367-
dotnet-version: ${{env.DOTNET_VERSION}}
368-
dotnet-quality: "ga"
371+
# use global.json to install the correct version
372+
global-json-file: global.json
369373
- run: src/ci/dotnet-fuzzing-tools.sh
370374
shell: bash
371375
- uses: actions/upload-artifact@v3
@@ -379,8 +383,8 @@ jobs:
379383
- name: Setup .NET Core SDK
380384
uses: actions/setup-dotnet@v3
381385
with:
382-
dotnet-version: ${{env.DOTNET_VERSION}}
383-
dotnet-quality: "ga"
386+
# use global.json to install the correct version
387+
global-json-file: global.json
384388
- run: src/ci/dotnet-fuzzing-tools.ps1
385389
shell: pwsh
386390
- uses: actions/upload-artifact@v3
@@ -471,8 +475,10 @@ jobs:
471475
cp artifacts/agent-MINGW*/onefuzz-task.exe src/deployment/tools/win64/
472476
cp artifacts/agent-MINGW*/onefuzz_agent.pdb src/deployment/tools/win64/
473477
cp artifacts/agent-MINGW*/onefuzz_task.pdb src/deployment/tools/win64/
474-
cp artifacts/agent-Linux/onefuzz-agent src/deployment/tools/linux/
475-
cp artifacts/agent-Linux/onefuzz-task src/deployment/tools/linux/
478+
cp artifacts/agent-Linux-x86_64/onefuzz-agent src/deployment/tools/linux/
479+
cp artifacts/agent-Linux-x86_64/onefuzz-task src/deployment/tools/linux/
480+
# cp artifacts/agent-Linux-aarch64/onefuzz-agent src/deployment/tools/linux-aarch64/
481+
# cp artifacts/agent-Linux-aarch64/onefuzz-task src/deployment/tools/linux-aarch64/
476482
477483
cp artifacts/proxy/onefuzz-proxy-manager src/deployment/tools/linux/
478484
cp artifacts/service/api-service.zip src/deployment

CHANGELOG.md

Lines changed: 94 additions & 12 deletions
Large diffs are not rendered by default.

CURRENT_VERSION

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
7.0.0
1+
8.0.0

global.json

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
{
2+
"sdk": {
3+
"version": "7.0.100",
4+
"rollForward": "latestFeature"
5+
}
6+
}

src/ApiService/ApiService/ApiService.csproj

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,8 @@
2525

2626
<PackageReference Include="Microsoft.Azure.Functions.Worker.Sdk" Version="1.7.0" OutputItemType="Analyzer" />
2727
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.10.0" />
28-
<PackageReference Include="Azure.Data.Tables" Version="12.5.0" />
28+
<PackageReference Include="Azure.Data.Tables" Version="12.8.0" />
2929
<PackageReference Include="Azure.ResourceManager.Compute" Version="1.0.0-beta.8" />
30-
<PackageReference Include="Azure.Core" Version="1.25.0" />
3130
<PackageReference Include="Azure.Identity" Version="1.8.2" />
3231
<PackageReference Include="Azure.Messaging.EventGrid" Version="4.10.0" />
3332
<PackageReference Include="Azure.ResourceManager" Version="1.3.1" />
@@ -39,7 +38,7 @@
3938
<PackageReference Include="Microsoft.Graph" Version="4.37.0" />
4039
<PackageReference Include="Microsoft.Identity.Client" Version="4.52.0" />
4140
<PackageReference Include="Microsoft.Identity.Web.TokenCache" Version="2.7.0" />
42-
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.22.1" />
41+
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.29.0" />
4342
<PackageReference Include="System.Linq.Async" Version="6.0.1" />
4443
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.SignalRService" Version="1.7.0" />
4544
<PackageReference Include="System.Text.RegularExpressions" Version="4.3.1" />

src/ApiService/ApiService/Functions/AgentCommands.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
3131
}
3232
var nodeCommand = request.OkV;
3333

34-
var message = await _context.NodeMessageOperations.GetMessage(nodeCommand.MachineId).FirstOrDefaultAsync();
34+
var message = await _context.NodeMessageOperations.GetMessage(nodeCommand.MachineId);
3535
if (message != null) {
3636
var command = message.Message;
3737
var messageId = message.MessageId;

src/ApiService/ApiService/Functions/Node.cs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
4646

4747
var (tasks, messages) = await (
4848
_context.NodeTasksOperations.GetByMachineId(machineId).ToListAsync().AsTask(),
49-
_context.NodeMessageOperations.GetMessage(machineId).ToListAsync().AsTask());
49+
_context.NodeMessageOperations.GetMessages(machineId).ToListAsync().AsTask());
5050

5151
var commands = messages.Select(m => m.Message).ToList();
5252
return await RequestHandling.Ok(req, NodeToNodeSearchResult(node with { Tasks = tasks, Messages = commands }));

src/ApiService/ApiService/onefuzzlib/NodeMessageOperations.cs

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using ApiService.OneFuzzLib.Orm;
1+
using System.Threading.Tasks;
2+
using ApiService.OneFuzzLib.Orm;
23
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
34

45
namespace Microsoft.OneFuzz.Service;
@@ -14,7 +15,9 @@ public NodeMessage(Guid machineId, NodeCommand message) : this(machineId, NewSor
1415
};
1516

1617
public interface INodeMessageOperations : IOrm<NodeMessage> {
17-
IAsyncEnumerable<NodeMessage> GetMessage(Guid machineId);
18+
IAsyncEnumerable<NodeMessage> GetMessages(Guid machineId);
19+
20+
Async.Task<NodeMessage?> GetMessage(Guid machineId);
1821
Async.Task ClearMessages(Guid machineId);
1922

2023
Async.Task SendMessage(Guid machineId, NodeCommand message, string? messageId = null);
@@ -25,7 +28,7 @@ public class NodeMessageOperations : Orm<NodeMessage>, INodeMessageOperations {
2528
public NodeMessageOperations(ILogTracer log, IOnefuzzContext context)
2629
: base(log, context) { }
2730

28-
public IAsyncEnumerable<NodeMessage> GetMessage(Guid machineId)
31+
public IAsyncEnumerable<NodeMessage> GetMessages(Guid machineId)
2932
=> QueryAsync(Query.PartitionKey(machineId.ToString()));
3033

3134
public async Async.Task ClearMessages(Guid machineId) {
@@ -45,4 +48,7 @@ public async Async.Task SendMessage(Guid machineId, NodeCommand message, string?
4548
_logTracer.WithHttpStatus(r.ErrorV).Error($"failed to insert message with id: {messageId:Tag:MessageId} for machine id: {machineId:Tag:MachineId} message: {message:Tag:Message}");
4649
}
4750
}
51+
52+
public async Task<NodeMessage?> GetMessage(Guid machineId)
53+
=> await QueryAsync(Query.PartitionKey(machineId.ToString()), maxPerPage: 1).FirstOrDefaultAsync();
4854
}

src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs

Lines changed: 55 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@ public override string ConvertName(string name) {
8888
}
8989
public class EntityConverter {
9090

91+
private const int MAX_DESERIALIZATION_RECURSION_DEPTH = 100;
9192
private readonly ConcurrentDictionary<Type, EntityInfo> _cache;
9293
private static readonly JsonSerializerOptions _options = new() {
9394
PropertyNamingPolicy = new OnefuzzNamingPolicy(),
@@ -124,8 +125,8 @@ public static JsonSerializerOptions GetJsonSerializerOptions() {
124125
}
125126

126127
private static IEnumerable<EntityProperty> GetEntityProperties<T>(ParameterInfo parameterInfo) {
127-
var name = parameterInfo.Name.EnsureNotNull($"Invalid paramter {parameterInfo}");
128-
var parameterType = parameterInfo.ParameterType.EnsureNotNull($"Invalid paramter {parameterInfo}");
128+
var name = parameterInfo.Name.EnsureNotNull($"Invalid parameter {parameterInfo}");
129+
var parameterType = parameterInfo.ParameterType.EnsureNotNull($"Invalid parameter {parameterInfo}");
129130
var isRowkey = parameterInfo.GetCustomAttribute(typeof(RowKeyAttribute)) != null;
130131
var isPartitionkey = parameterInfo.GetCustomAttribute(typeof(PartitionKeyAttribute)) != null;
131132

@@ -135,7 +136,7 @@ private static IEnumerable<EntityProperty> GetEntityProperties<T>(ParameterInfo
135136

136137
(TypeDiscrimnatorAttribute, ITypeProvider)? discriminator = null;
137138
if (discriminatorAttribute != null) {
138-
var t = (ITypeProvider)(Activator.CreateInstance(discriminatorAttribute.ConverterType) ?? throw new Exception("unable to retrive the type provider"));
139+
var t = (ITypeProvider)(Activator.CreateInstance(discriminatorAttribute.ConverterType) ?? throw new Exception("unable to retrieve the type provider"));
139140
discriminator = (discriminatorAttribute, t);
140141
}
141142

@@ -222,7 +223,7 @@ public TableEntity ToTableEntity<T>(T typedEntity) where T : EntityBase {
222223
}
223224

224225

225-
private object? GetFieldValue(EntityInfo info, string name, TableEntity entity) {
226+
private object? GetFieldValue(EntityInfo info, string name, TableEntity entity, int iterationCount) {
226227
var ef = info.properties[name].First();
227228
if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey) {
228229
// partition & row keys must always be strings
@@ -285,7 +286,23 @@ public TableEntity ToTableEntity<T>(T typedEntity) where T : EntityBase {
285286
var outputType = ef.type;
286287
if (ef.discriminator != null) {
287288
var (attr, typeProvider) = ef.discriminator.Value;
288-
var v = GetFieldValue(info, attr.FieldName, entity) ?? throw new Exception($"No value for {attr.FieldName}");
289+
if (iterationCount > MAX_DESERIALIZATION_RECURSION_DEPTH) {
290+
var tags = GenerateTableEntityTags(entity);
291+
tags.AddRange(new (string, string)[] {
292+
("outputType", outputType?.Name ?? string.Empty),
293+
("fieldName", fieldName)
294+
});
295+
throw new OrmMaxRecursionDepthReachedException($"MAX_DESERIALIZATION_RECURSION_DEPTH reached. Too many iterations deserializing {info.type}. {PrintTags(tags)}");
296+
}
297+
if (attr.FieldName == name) {
298+
var tags = GenerateTableEntityTags(entity);
299+
tags.AddRange(new (string, string)[] {
300+
("outputType", outputType?.Name ?? string.Empty),
301+
("fieldName", fieldName)
302+
});
303+
throw new OrmInvalidDiscriminatorFieldException($"Discriminator field is the same as the field being deserialized {name}. {PrintTags(tags)}");
304+
}
305+
var v = GetFieldValue(info, attr.FieldName, entity, ++iterationCount) ?? throw new Exception($"No value for {attr.FieldName}");
289306
outputType = typeProvider.GetTypeInfo(v);
290307
}
291308

@@ -302,8 +319,13 @@ public TableEntity ToTableEntity<T>(T typedEntity) where T : EntityBase {
302319
return JsonSerializer.Deserialize(value, outputType, options: _options);
303320
}
304321
}
305-
} catch (Exception ex) {
306-
throw new InvalidOperationException($"Unable to get value for property '{name}' (entity field '{fieldName}')", ex);
322+
} catch (Exception ex)
323+
when (ex is not OrmException) {
324+
var tags = GenerateTableEntityTags(entity);
325+
tags.AddRange(new (string, string)[] {
326+
("fieldName", fieldName)
327+
});
328+
throw new InvalidOperationException($"Unable to get value for property '{name}' (entity field '{fieldName}'). {PrintTags(tags)}", ex);
307329
}
308330
}
309331

@@ -313,7 +335,7 @@ public T ToRecord<T>(TableEntity entity) where T : EntityBase {
313335

314336
object?[] parameters;
315337
try {
316-
parameters = entityInfo.properties.Select(grouping => GetFieldValue(entityInfo, grouping.Key, entity)).ToArray();
338+
parameters = entityInfo.properties.Select(grouping => GetFieldValue(entityInfo, grouping.Key, entity, 0)).ToArray();
317339
} catch (Exception ex) {
318340
throw new InvalidOperationException($"Unable to extract properties from TableEntity for {typeof(T)}", ex);
319341
}
@@ -361,6 +383,31 @@ public T ToRecord<T>(TableEntity entity) where T : EntityBase {
361383
return Expression.Lambda<Func<T, object?>>(call, paramter).Compile();
362384
}
363385

386+
private static List<(string, string)> GenerateTableEntityTags(TableEntity entity) {
387+
var entityKeys = string.Join(',', entity.Keys);
388+
var partitionKey = entity.ContainsKey(EntityPropertyKind.PartitionKey.ToString()) ? entity.GetString(EntityPropertyKind.PartitionKey.ToString()) : string.Empty;
389+
var rowKey = entity.ContainsKey(EntityPropertyKind.RowKey.ToString()) ? entity.GetString(EntityPropertyKind.RowKey.ToString()) : string.Empty;
364390

391+
return new List<(string, string)> {
392+
("entityKeys", entityKeys),
393+
("partitionKey", partitionKey),
394+
("rowKey", rowKey)
395+
};
396+
}
397+
398+
private static string PrintTags(List<(string, string)>? tags) {
399+
return tags != null ? string.Join(", ", tags.Select(x => $"{x.Item1}={x.Item2}")) : string.Empty;
400+
}
401+
}
402+
403+
public class OrmInvalidDiscriminatorFieldException : OrmException {
404+
public OrmInvalidDiscriminatorFieldException(string message) : base(message) { }
405+
}
406+
407+
public class OrmMaxRecursionDepthReachedException : OrmException {
408+
public OrmMaxRecursionDepthReachedException(string message) : base(message) { }
409+
}
365410

411+
public class OrmException : Exception {
412+
public OrmException(string message) : base(message) { }
366413
}

src/ApiService/ApiService/onefuzzlib/orm/Orm.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
namespace ApiService.OneFuzzLib.Orm {
1313
public interface IOrm<T> where T : EntityBase {
1414
Task<TableClient> GetTableClient(string table, ResourceIdentifier? accountId = null);
15-
IAsyncEnumerable<T> QueryAsync(string? filter = null);
15+
IAsyncEnumerable<T> QueryAsync(string? filter = null, int? maxPerPage = null);
1616

1717
Task<T> GetEntityAsync(string partitionKey, string rowKey);
1818
Task<ResultVoid<(HttpStatusCode Status, string Reason)>> Insert(T entity);
@@ -49,14 +49,14 @@ public Orm(ILogTracer logTracer, IOnefuzzContext context) {
4949
_entityConverter = _context.EntityConverter;
5050
}
5151

52-
public async IAsyncEnumerable<T> QueryAsync(string? filter = null) {
52+
public async IAsyncEnumerable<T> QueryAsync(string? filter = null, int? maxPerPage = null) {
5353
var tableClient = await GetTableClient(typeof(T).Name);
5454

5555
if (filter == "") {
5656
filter = null;
5757
}
5858

59-
await foreach (var x in tableClient.QueryAsync<TableEntity>(filter).Select(x => _entityConverter.ToRecord<T>(x))) {
59+
await foreach (var x in tableClient.QueryAsync<TableEntity>(filter: filter, maxPerPage: maxPerPage).Select(x => _entityConverter.ToRecord<T>(x))) {
6060
yield return x;
6161
}
6262
}

0 commit comments

Comments
 (0)