Migrating QueueTaskHeartbeat (#1777)

* Migrating QueueTaskHeartbeat

* changing the name of the input queue

* rename type alias Tasks to Async

* Fix property casing

* fixing types

* Removing IStorageProvider

* fix function name

* address PR comments

Author: chkeita

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -2,9 +2,8 @@
 using System;
 using System.Collections.Generic;
 using PoolName = System.String;
-using Endpoint = System.String;
-using GroupId = System.Guid;
-using PrincipalId = System.Guid;
+using Region = System.String;
+using Container = System.String;
 
 namespace Microsoft.OneFuzz.Service;
 
@@ -101,7 +100,7 @@ public enum NodeState
 
 public record ProxyHeartbeat
 (
-    string Region,
+    Region Region,
     Guid ProxyId,
     List<ProxyForward> Forwards,
     DateTimeOffset TimeStamp
@@ -125,7 +124,7 @@ bool DebugKeepNode
 
 public partial record ProxyForward
 (
-    [PartitionKey] string Region,
+    [PartitionKey] Region Region,
     [RowKey] int DstPort,
     int SrcPort,
     string DstIp
@@ -135,7 +134,7 @@ public partial record ProxyConfig
 (
     Uri Url,
     string Notification,
-    string Region,
+    Region Region,
     Guid? ProxyId,
     List<ProxyForward> Forwards,
     string InstanceTelemetryKey,
@@ -145,7 +144,7 @@ string MicrosoftTelemetryKey
 
 public partial record Proxy
 (
-    [PartitionKey] string Region,
+    [PartitionKey] Region Region,
     [RowKey] Guid ProxyId,
     DateTimeOffset? CreatedTimestamp,
     VmState State,
@@ -170,9 +169,72 @@ String InstanceName
 ) : EntityBase();
 
 
-//record AnyHttpUrl(AnyUrl):
-//    allowed_schemes = {'http', 'https
-//
+public record TaskDetails(
+
+    TaskType Type,
+    int Duration,
+    string? TargetExe,
+    Dictionary<string, string>? TargetEnv,
+    List<string>? TargetOptions,
+    int? TargetWorkers,
+    bool? TargetOptionsMerge,
+    bool? CheckAsanLog,
+    bool? CheckDebugger,
+    int? CheckRetryCount,
+    bool? CheckFuzzerHelp,
+    bool? ExpectCrashOnFailure,
+    bool? RenameOutput,
+    string? SupervisorExe,
+    Dictionary<string, string>? SupervisorEnv,
+    List<string>? SupervisorOptions,
+    string? SupervisorInputMarker,
+    string? GeneratorExe,
+    Dictionary<string, string>? GeneratorEnv,
+    List<string>? GeneratorOptions,
+    string? AnalyzerExe,
+    Dictionary<string, string>? AnalyzerEnv,
+    List<string> AnalyzerOptions,
+    ContainerType? WaitForFiles,
+    string? StatsFile,
+    StatsFormat? StatsFormat,
+    bool? RebootAfterSetup,
+    int? TargetTimeout,
+    int? EnsembleSyncDelay,
+    bool? PreserveExistingOutputs,
+    List<string>? ReportList,
+    int? MinimizedStackDepth,
+    string? CoverageFilter
+);
+
+public record TaskVm(
+    Region Region,
+    string Sku,
+    string Image,
+    int Count,
+    bool SpotInstance,
+    bool? RebootAfterSetup
+);
+
+public record TaskPool(
+    int Count,
+    PoolName PoolName
+);
+
+public record TaskContainers(
+    ContainerType Type,
+    Container Name
+);
+public record TaskConfig(
+   Guid JobId,
+   List<Guid>? PrereqTasks,
+   TaskDetails Task,
+   TaskVm? Vm,
+   TaskPool? Pool,
+   List<TaskContainers>? Containers,
+   Dictionary<string, string>? Tags,
+   List<TaskDebugFlag>? Debug,
+   bool? Colocate
+   );
 
 
 public record TaskEventSummary(
@@ -204,118 +266,5 @@ public record Task(
 {
     List<TaskEventSummary> Events { get; set; } = new List<TaskEventSummary>();
     List<NodeAssignment> Nodes { get; set; } = new List<NodeAssignment>();
-}
-public record AzureSecurityExtensionConfig();
-public record GenevaExtensionConfig();
-
-
-public record KeyvaultExtensionConfig(
-    string KeyVaultName,
-    string CertName,
-    string CertPath,
-    string ExtensionStore
-);
-
-public record AzureMonitorExtensionConfig(
-    string ConfigVersion,
-    string Moniker,
-    string Namespace,
-    [property: JsonPropertyName("monitoringGSEnvironment")] string MonitoringGSEnvironment,
-    [property: JsonPropertyName("monitoringGCSAccount")] string MonitoringGCSAccount,
-    [property: JsonPropertyName("monitoringGCSAuthId")] string MonitoringGCSAuthId,
-    [property: JsonPropertyName("monitoringGCSAuthIdType")] string MonitoringGCSAuthIdType
-);
 
-public record AzureVmExtensionConfig(
-    KeyvaultExtensionConfig? Keyvault,
-    AzureMonitorExtensionConfig AzureMonitor
-);
-
-public record NetworkConfig(
-    string AddressSpace,
-    string Subnet
-)
-{
-    public NetworkConfig() : this("10.0.0.0/8", "10.0.0.0/16") { }
-}
-
-public record NetworkSecurityGroupConfig(
-    string[] AllowedServiceTags,
-    string[] AllowedIps
-)
-{
-    public NetworkSecurityGroupConfig() : this(Array.Empty<string>(), Array.Empty<string>()) { }
-}
-
-public record ApiAccessRule(
-    string[] Methods,
-    Guid[] AllowedGroups
-);
-
-public record InstanceConfig
-(
-    [PartitionKey, RowKey] string InstanceName,
-    //# initial set of admins can only be set during deployment.
-    //# if admins are set, only admins can update instance configs.
-    Guid[]? Admins,
-    //# if set, only admins can manage pools or scalesets
-    bool AllowPoolManagement,
-    string[] AllowedAadTenants,
-    NetworkConfig NetworkConfig,
-    NetworkSecurityGroupConfig ProxyNsgConfig,
-    AzureVmExtensionConfig? Extensions,
-    string ProxyVmSku,
-    IDictionary<Endpoint, ApiAccessRule>? ApiAccessRules,
-    IDictionary<PrincipalId, GroupId[]>? GroupMembership,
-
-    IDictionary<string, string>? VmTags,
-    IDictionary<string, string>? VmssTags
-) : EntityBase()
-{
-    public InstanceConfig(string instanceName) : this(
-        instanceName,
-        null,
-        true,
-        Array.Empty<string>(),
-        new NetworkConfig(),
-        new NetworkSecurityGroupConfig(),
-        null,
-        "Standard_B2s",
-        null,
-        null,
-        null,
-        null)
-    { }
-
-    public List<Guid>? CheckAdmins(List<Guid>? value)
-    {
-        if (value is not null && value.Count == 0)
-        {
-            throw new ArgumentException("admins must be null or contain at least one UUID");
-        }
-        else
-        {
-            return value;
-        }
-    }
-
-
-    //# At the moment, this only checks allowed_aad_tenants, however adding
-    //# support for 3rd party JWT validation is anticipated in a future release.
-    public ResultOk<List<string>> CheckInstanceConfig()
-    {
-        List<string> errors = new();
-        if (AllowedAadTenants.Length == 0)
-        {
-            errors.Add("allowed_aad_tenants must not be empty");
-        }
-        if (errors.Count == 0)
-        {
-            return ResultOk<List<string>>.Ok();
-        }
-        else
-        {
-            return ResultOk<List<string>>.Error(errors);
-        }
-    }
 }

src/ApiService/ApiService/Program.cs

@@ -5,9 +5,10 @@
 using System.Collections.Generic;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.DependencyInjection;
-using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 using ApiService.OneFuzzLib;
 
+
+
 namespace Microsoft.OneFuzz.Service;
 
 public class Program
@@ -65,11 +66,11 @@ public static void Main()
         .ConfigureServices((context, services) =>
             services
             .AddSingleton<ILogTracerFactory>(_ => new LogTracerFactory(GetLoggers()))
-            .AddSingleton<IStorageProvider>(_ => new StorageProvider(EnvironmentVariables.OneFuzz.FuncStorage ?? throw new InvalidOperationException("Missing account id")))
             .AddSingleton<INodeOperations, NodeOperations>()
             .AddSingleton<IEvents, Events>()
             .AddSingleton<IWebhookOperations, WebhookOperations>()
             .AddSingleton<IWebhookMessageLogOperations, WebhookMessageLogOperations>()
+            .AddSingleton<ITaskOperations, TaskOperations>()
             .AddSingleton<IQueue, Queue>()
             .AddSingleton<ICreds>(_ => new Creds())
             .AddSingleton<IStorage, Storage>()

src/ApiService/ApiService/QueueFileChanges.cs

@@ -14,14 +14,12 @@ public class QueueFileChanges
     const int MAX_DEQUEUE_COUNT = 5;
 
     private readonly ILogTracerFactory _loggerFactory;
-    private readonly IStorageProvider _storageProvider;
 
     private readonly IStorage _storage;
 
-    public QueueFileChanges(ILogTracerFactory loggerFactory, IStorageProvider storageProvider, IStorage storage)
+    public QueueFileChanges(ILogTracerFactory loggerFactory, IStorage storage)
     {
         _loggerFactory = loggerFactory;
-        _storageProvider = storageProvider;
         _storage = storage;
     }
 
@@ -52,7 +50,7 @@ public Async.Task Run(
         }
 
         file_added(log, fileChangeEvent, lastTry);
-        return Task.CompletedTask;
+        return Async.Task.CompletedTask;
     }
 
     private void file_added(ILogTracer log, Dictionary<string, string> fileChangeEvent, bool failTaskOnTransientError)

src/ApiService/ApiService/QueueProxyHeartbeat.cs

@@ -1,7 +1,6 @@
 using System;
 using Microsoft.Azure.Functions.Worker;
 using System.Text.Json;
-using System.Threading.Tasks;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 
 namespace Microsoft.OneFuzz.Service;
@@ -19,7 +18,7 @@ public QueueProxyHearbeat(ILogTracerFactory loggerFactory, IProxyOperations prox
     }
 
     [Function("QueueProxyHearbeat")]
-    public async Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg)
+    public async Async.Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg)
     {
         var log = _loggerFactory.MakeLogTracer(Guid.NewGuid());
 

src/ApiService/ApiService/onefuzzlib/Events.cs

@@ -5,7 +5,6 @@
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;
-using System.Threading.Tasks;
 
 namespace Microsoft.OneFuzz.Service
 {