1.0.8: Adding Experimental feature flags for the new experimental features

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/ExperimentalFeatures.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft. All rights reserved.
+namespace Microsoft.Azure.Devices.Edge.Agent.Service
+{
+    using Microsoft.Azure.Devices.Edge.Storage;
+    using Microsoft.Extensions.Configuration;
+    using Microsoft.Extensions.Logging;
+
+    public class ExperimentalFeatures
+    {
+        ExperimentalFeatures(bool enabled, bool disableCloudSubscriptions, bool enableUploadLogs)
+        {
+            this.Enabled = enabled;
+            this.DisableCloudSubscriptions = disableCloudSubscriptions;
+            this.EnableUploadLogs = enableUploadLogs;
+        }
+
+        public static ExperimentalFeatures Init(IConfiguration experimentalFeaturesConfig, ILogger logger)
+        {
+            bool enabled = experimentalFeaturesConfig.GetValue("enabled", false);
+            bool disableCloudSubscriptions = enabled && experimentalFeaturesConfig.GetValue("disableCloudSubscriptions", false);
+            bool enableUploadLogs = enabled && experimentalFeaturesConfig.GetValue("enableUploadLogs", false);
+            var experimentalFeatures = new ExperimentalFeatures(enabled, disableCloudSubscriptions, enableUploadLogs);
+            logger.LogInformation($"Experimental features configuration: {experimentalFeatures.ToJson()}");
+            return experimentalFeatures;
+        }
+
+        public bool Enabled { get; }
+
+        public bool DisableCloudSubscriptions { get; }
+
+        public bool EnableUploadLogs { get; }
+    }
+}

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Program.cs

@@ -107,6 +107,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                 bool closeOnIdleTimeout = configuration.GetValue(Constants.CloseOnIdleTimeout, false);
                 int idleTimeoutSecs = configuration.GetValue(Constants.IdleTimeoutSecs, 300);
                 TimeSpan idleTimeout = TimeSpan.FromSeconds(idleTimeoutSecs);
+                ExperimentalFeatures experimentalFeatures = ExperimentalFeatures.Init(configuration.GetSection("experimentalFeatures"), logger);
                 string iothubHostname;
                 string deviceId;
                 switch (mode.ToLowerInvariant())
@@ -142,7 +143,6 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                     case "twin":
                         bool enableStreams = configuration.GetValue(Constants.EnableStreams, false);
                         int requestTimeoutSecs = configuration.GetValue(Constants.RequestTimeoutSecs, 600);
-                        bool disableSubscriptions = configuration.GetValue(Constants.DisableCloudSubscriptions, false);
                         builder.RegisterModule(
                             new TwinConfigSourceModule(
                                 iothubHostname,
@@ -153,7 +153,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                                 TimeSpan.FromSeconds(configRefreshFrequencySecs),
                                 enableStreams,
                                 TimeSpan.FromSeconds(requestTimeoutSecs),
-                                !disableSubscriptions));
+                                experimentalFeatures));
                         break;
 
                     case "local":

edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/TwinConfigSourceModule.cs

@@ -31,7 +31,7 @@ public class TwinConfigSourceModule : Module
         readonly string iotHubHostName;
         readonly bool enableStreams;
         readonly TimeSpan requestTimeout;
-        readonly bool enableSubscriptions;
+        readonly ExperimentalFeatures experimentalFeatures;
 
         public TwinConfigSourceModule(
             string iotHubHostname,
@@ -42,7 +42,7 @@ public TwinConfigSourceModule(
             TimeSpan configRefreshFrequency,
             bool enableStreams,
             TimeSpan requestTimeout,
-            bool enableSubscriptions)
+            ExperimentalFeatures experimentalFeatures)
         {
             this.iotHubHostName = Preconditions.CheckNonWhiteSpace(iotHubHostname, nameof(iotHubHostname));
             this.deviceId = Preconditions.CheckNonWhiteSpace(deviceId, nameof(deviceId));
@@ -52,7 +52,7 @@ public TwinConfigSourceModule(
             this.configRefreshFrequency = configRefreshFrequency;
             this.enableStreams = enableStreams;
             this.requestTimeout = requestTimeout;
-            this.enableSubscriptions = enableSubscriptions;
+            this.experimentalFeatures = experimentalFeatures;
         }
 
         protected override void Load(ContainerBuilder builder)
@@ -84,10 +84,15 @@ protected override void Load(ContainerBuilder builder)
                     ILogsProvider logsProvider = await logsProviderTask;
                     var requestHandlers = new List<IRequestHandler>
                     {
-                        new PingRequestHandler(),
-                        new LogsUploadRequestHandler(logsUploader, logsProvider, runtimeInfoProvider),
-                        new TaskStatusRequestHandler()
+                        new PingRequestHandler()
                     };
+
+                    if (this.experimentalFeatures.EnableUploadLogs)
+                    {
+                        requestHandlers.Add(new LogsUploadRequestHandler(logsUploader, logsProvider, runtimeInfoProvider));
+                        requestHandlers.Add(new TaskStatusRequestHandler());
+                    }
+
                     return new RequestManager(requestHandlers, this.requestTimeout) as IRequestManager;
                 })
                 .As<Task<IRequestManager>>()
@@ -106,7 +111,8 @@ protected override void Load(ContainerBuilder builder)
                     var deviceClientprovider = c.Resolve<IModuleClientProvider>();
                     var requestManagerTask = c.Resolve<Task<IRequestManager>>();
                     IRequestManager requestManager = await requestManagerTask;
-                    IEdgeAgentConnection edgeAgentConnection = new EdgeAgentConnection(deviceClientprovider, serde, requestManager, this.enableSubscriptions, this.configRefreshFrequency);
+                    bool enableSubscriptions = !this.experimentalFeatures.DisableCloudSubscriptions;
+                    IEdgeAgentConnection edgeAgentConnection = new EdgeAgentConnection(deviceClientprovider, serde, requestManager, enableSubscriptions, this.configRefreshFrequency);
                     return edgeAgentConnection;
                 })
                 .As<Task<IEdgeAgentConnection>>()

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.CloudProxy/DeviceConnectivityManager.cs

@@ -282,7 +282,7 @@ internal static void SetConnectionManager()
 
             internal static void Created(TimeSpan connectedCheckFrequency, TimeSpan disconnectedCheckFrequency)
             {
-                Log.LogDebug((int)EventIds.Created, Invariant($"Created DeviceConnectivityManager with connected check frequency {connectedCheckFrequency} and disconnected check frequency {disconnectedCheckFrequency}"));
+                Log.LogInformation((int)EventIds.Created, Invariant($"Created DeviceConnectivityManager with connected check frequency {connectedCheckFrequency} and disconnected check frequency {disconnectedCheckFrequency}"));
             }
         }
     }

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Core/HubCoreEventIds.cs

@@ -20,6 +20,7 @@ public static class HubCoreEventIds
         public const int DeviceScopeAuthenticator = EventIdStart + 1400;
         public const int SubscriptionProcessor = EventIdStart + 1500;
         public const int RetryingCloudProxy = EventIdStart + 1600;
+        public const int NullDeviceConnectivityManager = EventIdStart + 1700;
         const int EventIdStart = 1000;
     }
 }

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Core/NullDeviceConnectivityManager.cs

@@ -3,9 +3,16 @@ namespace Microsoft.Azure.Devices.Edge.Hub.Core
 {
     using System;
     using System.Threading.Tasks;
+    using Microsoft.Azure.Devices.Edge.Util;
+    using Microsoft.Extensions.Logging;
 
     public class NullDeviceConnectivityManager : IDeviceConnectivityManager
     {
+        public NullDeviceConnectivityManager()
+        {
+            Events.Created();
+        }
+
         public event EventHandler DeviceConnected
         {
             add { }
@@ -21,5 +28,21 @@ public event EventHandler DeviceDisconnected
         public Task CallSucceeded() => Task.CompletedTask;
 
         public Task CallTimedOut() => Task.CompletedTask;
+
+        static class Events
+        {
+            const int IdStart = HubCoreEventIds.NullDeviceConnectivityManager;
+            static readonly ILogger Log = Logger.Factory.CreateLogger<NullDeviceConnectivityManager>();
+
+            enum EventIds
+            {
+                Created = IdStart
+            }
+
+            internal static void Created()
+            {
+                Log.LogInformation((int)EventIds.Created, $"Device connectivity check disabled");
+            }
+        }
     }
 }

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/DependencyManager.cs

@@ -69,15 +69,18 @@ public void Register(ContainerBuilder builder)
                     var loggerFactory = c.Resolve<ILoggerFactory>();
                     InternalLoggerFactory.DefaultFactory = loggerFactory;
 
-                    var eventListener = new LoggerEventListener(loggerFactory.CreateLogger("ProtocolGateway"));
+                    var eventListener = new LoggerEventListener(loggerFactory.CreateLogger("EdgeHub"));
                     eventListener.EnableEvents(CommonEventSource.Log, EventLevel.Informational);
                 });
 
             bool optimizeForPerformance = this.configuration.GetValue("OptimizeForPerformance", true);
             (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath) storeAndForward = this.GetStoreAndForwardConfiguration();
 
+            IConfiguration configuration = this.configuration.GetSection("experimentalFeatures");
+            ExperimentalFeatures experimentalFeatures = ExperimentalFeatures.Init(configuration);
+
             this.RegisterCommonModule(builder, optimizeForPerformance, storeAndForward);
-            this.RegisterRoutingModule(builder, storeAndForward);
+            this.RegisterRoutingModule(builder, storeAndForward, experimentalFeatures);
             this.RegisterMqttModule(builder, storeAndForward, optimizeForPerformance);
             this.RegisterAmqpModule(builder);
             builder.RegisterModule(new HttpModule());
@@ -107,7 +110,10 @@ void RegisterMqttModule(ContainerBuilder builder, (bool isEnabled, bool usePersi
             builder.RegisterModule(new MqttModule(mqttSettingsConfiguration, topics, this.serverCertificate, storeAndForward.isEnabled, clientCertAuthEnabled, optimizeForPerformance));
         }
 
-        void RegisterRoutingModule(ContainerBuilder builder, (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath) storeAndForward)
+        void RegisterRoutingModule(
+            ContainerBuilder builder,
+            (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath) storeAndForward,
+            ExperimentalFeatures experimentalFeatures)
         {
             var routes = this.configuration.GetSection("routes").Get<Dictionary<string, string>>();
             int connectionPoolSize = this.configuration.GetValue<int>("IotHubConnectionPoolSize");
@@ -133,8 +139,6 @@ void RegisterRoutingModule(ContainerBuilder builder, (bool isEnabled, bool usePe
             int maxUpstreamBatchSize = this.configuration.GetValue("MaxUpstreamBatchSize", 10);
             int upstreamFanOutFactor = this.configuration.GetValue("UpstreamFanOutFactor", 10);
             bool encryptTwinStore = this.configuration.GetValue("EncryptTwinStore", false);
-            bool disableCloudSubscriptions = this.configuration.GetValue("DisableCloudSubscriptions", false);
-            bool enableConnectivityCheck = this.configuration.GetValue("EnableConnectivityCheck", true);
             int configUpdateFrequencySecs = this.configuration.GetValue("ConfigRefreshFrequencySecs", 3600);
             TimeSpan configUpdateFrequency = TimeSpan.FromSeconds(configUpdateFrequencySecs);
 
@@ -162,9 +166,8 @@ void RegisterRoutingModule(ContainerBuilder builder, (bool isEnabled, bool usePe
                     maxUpstreamBatchSize,
                     upstreamFanOutFactor,
                     encryptTwinStore,
-                    disableCloudSubscriptions,
                     configUpdateFrequency,
-                    enableConnectivityCheck));
+                    experimentalFeatures));
         }
 
         void RegisterCommonModule(ContainerBuilder builder, bool optimizeForPerformance, (bool isEnabled, bool usePersistentStorage, StoreAndForwardConfiguration config, string storagePath) storeAndForward)

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/ExperimentalFeatures.cs

@@ -0,0 +1,29 @@
+// Copyright (c) Microsoft. All rights reserved.
+namespace Microsoft.Azure.Devices.Edge.Hub.Service
+{
+    using Microsoft.Extensions.Configuration;
+
+    public class ExperimentalFeatures
+    {
+        public ExperimentalFeatures(bool enabled, bool disableCloudSubscriptions, bool disableConnectivityCheck)
+        {
+            this.Enabled = enabled;
+            this.DisableCloudSubscriptions = disableCloudSubscriptions;
+            this.DisableConnectivityCheck = disableConnectivityCheck;
+        }
+
+        public static ExperimentalFeatures Init(IConfiguration experimentalFeaturesConfig)
+        {
+            bool enabled = experimentalFeaturesConfig.GetValue("enabled", false);
+            bool disableCloudSubscriptions = enabled && experimentalFeaturesConfig.GetValue("disableCloudSubscriptions", false);
+            bool disableConnectivityCheck = enabled && experimentalFeaturesConfig.GetValue("disableConnectivityCheck", false);
+            return new ExperimentalFeatures(enabled, disableCloudSubscriptions, disableConnectivityCheck);
+        }
+
+        public bool Enabled { get; }
+
+        public bool DisableCloudSubscriptions { get; }
+
+        public bool DisableConnectivityCheck { get; }
+    }
+}

edge-hub/src/Microsoft.Azure.Devices.Edge.Hub.Service/modules/RoutingModule.cs

@@ -49,9 +49,8 @@ public class RoutingModule : Module
         readonly int maxUpstreamBatchSize;
         readonly int upstreamFanOutFactor;
         readonly bool encryptTwinStore;
-        readonly bool disableCloudSubscriptions;
         readonly TimeSpan configUpdateFrequency;
-        readonly bool enableConnectivityChecks;
+        readonly ExperimentalFeatures experimentalFeatures;
 
         public RoutingModule(
             string iotHubName,
@@ -76,9 +75,8 @@ public RoutingModule(
             int maxUpstreamBatchSize,
             int upstreamFanOutFactor,
             bool encryptTwinStore,
-            bool disableCloudSubscriptions,
             TimeSpan configUpdateFrequency,
-            bool enableConnectivityChecks)
+            ExperimentalFeatures experimentalFeatures)
         {
             this.iotHubName = Preconditions.CheckNonWhiteSpace(iotHubName, nameof(iotHubName));
             this.edgeDeviceId = Preconditions.CheckNonWhiteSpace(edgeDeviceId, nameof(edgeDeviceId));
@@ -102,9 +100,8 @@ public RoutingModule(
             this.maxUpstreamBatchSize = maxUpstreamBatchSize;
             this.upstreamFanOutFactor = upstreamFanOutFactor;
             this.encryptTwinStore = encryptTwinStore;
-            this.disableCloudSubscriptions = disableCloudSubscriptions;
             this.configUpdateFrequency = configUpdateFrequency;
-            this.enableConnectivityChecks = enableConnectivityChecks;
+            this.experimentalFeatures = experimentalFeatures;
         }
 
         protected override void Load(ContainerBuilder builder)
@@ -179,9 +176,9 @@ protected override void Load(ContainerBuilder builder)
                     c =>
                     {
                         var edgeHubCredentials = c.ResolveNamed<IClientCredentials>("EdgeHubCredentials");
-                        IDeviceConnectivityManager deviceConnectivityManager = this.enableConnectivityChecks
-                            ? new DeviceConnectivityManager(this.connectivityCheckFrequency, TimeSpan.FromMinutes(2), edgeHubCredentials.Identity)
-                            : new NullDeviceConnectivityManager() as IDeviceConnectivityManager;
+                        IDeviceConnectivityManager deviceConnectivityManager = this.experimentalFeatures.DisableConnectivityCheck
+                            ? new NullDeviceConnectivityManager()
+                            : new DeviceConnectivityManager(this.connectivityCheckFrequency, TimeSpan.FromMinutes(2), edgeHubCredentials.Identity) as IDeviceConnectivityManager;
                         return deviceConnectivityManager;
                     })
                 .As<IDeviceConnectivityManager>()
@@ -462,7 +459,7 @@ protected override void Load(ContainerBuilder builder)
                     async c =>
                     {
                         var connectionManagerTask = c.Resolve<Task<IConnectionManager>>();
-                        if (this.disableCloudSubscriptions)
+                        if (this.experimentalFeatures.DisableCloudSubscriptions)
                         {
                             return new LocalSubscriptionProcessor(await connectionManagerTask) as ISubscriptionProcessor;
                         }

edge-hub/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/DependencyManager.cs

@@ -84,6 +84,8 @@ public void Register(ContainerBuilder builder)
             var mqttSettingsConfiguration = new Mock<IConfiguration>();
             mqttSettingsConfiguration.Setup(c => c.GetSection(It.IsAny<string>())).Returns(Mock.Of<IConfigurationSection>(s => s.Value == null));
 
+            var experimentalFeatures = new ExperimentalFeatures(true, false, false);
+
             builder.RegisterBuildCallback(
                 c =>
                 {
@@ -141,9 +143,8 @@ public void Register(ContainerBuilder builder)
                     10,
                     10,
                     false,
-                    false,
                     TimeSpan.FromHours(1),
-                    true));
+                    experimentalFeatures));
 
             builder.RegisterModule(new HttpModule());
             builder.RegisterModule(new MqttModule(mqttSettingsConfiguration.Object, topics, this.serverCertificate, false, false, false));