Add RocksDB_MaxManifestFileSize env var to Edge Agent and Edge Hub. (#…

…5610) Cherry-pick from master of 2c87863 and 2385a9b The RocksDB MANIFEST file stores the history of operations. When it reaches the configured max size, RocksDB rolls over to a new file and marks the previous file as obsolete, so that it will be deleted in the next compaction. Since the default max size in RocksDB is 1 GiB which might be too high for some users, this env var allows those users to set it lower.
Azure · Oct 4, 2021 · c9c4b29 · c9c4b29
1 parent 12e12cf
commit c9c4b29
Show file tree

Hide file tree

Showing 16 changed files with 259 additions and 110 deletions.
diff --git a/doc/EnvironmentVariables.md b/doc/EnvironmentVariables.md
diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/Constants.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Core/Constants.cs
@@ -99,6 +99,8 @@ public static class Constants
 
         public const string StorageMaxTotalWalSize = "RocksDB_MaxTotalWalSize";
 
+        public const string StorageMaxManifestFileSize = "RocksDB_MaxManifestFileSize";
+
         public const string StorageMaxOpenFiles = "RocksDB_MaxOpenFiles";
 
         public const string StorageLogLevel = "Storage_LogLevel";

diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Program.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/Program.cs
@@ -154,6 +154,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                 TimeSpan idleTimeout = TimeSpan.FromSeconds(idleTimeoutSecs);
                 experimentalFeatures = ExperimentalFeatures.Create(configuration.GetSection("experimentalFeatures"), logger);
                 Option<ulong> storageTotalMaxWalSize = GetConfigIfExists<ulong>(Constants.StorageMaxTotalWalSize, configuration, logger);
+                Option<ulong> storageMaxManifestFileSize = GetConfigIfExists<ulong>(Constants.StorageMaxManifestFileSize, configuration, logger);
                 Option<int> storageMaxOpenFiles = GetConfigIfExists<int>(Constants.StorageMaxOpenFiles, configuration, logger);
                 Option<StorageLogLevel> storageLogLevel = GetConfigIfExists<StorageLogLevel>(Constants.StorageLogLevel, configuration, logger);
                 string iothubHostname;
@@ -168,7 +169,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                         IotHubConnectionStringBuilder connectionStringParser = IotHubConnectionStringBuilder.Create(deviceConnectionString);
                         deviceId = connectionStringParser.DeviceId;
                         iothubHostname = connectionStringParser.HostName;
-                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxOpenFiles, storageLogLevel));
+                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxManifestFileSize, storageMaxOpenFiles, storageLogLevel));
                         builder.RegisterModule(new DockerModule(deviceConnectionString, edgeDeviceHostName, dockerUri, dockerAuthConfig, upstreamProtocol, proxy, productInfo, closeOnIdleTimeout, idleTimeout, useServerHeartbeat, backupConfigFilePath));
                         break;
 
@@ -181,7 +182,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                         string moduleGenerationId = configuration.GetValue<string>(Constants.EdgeletModuleGenerationIdVariableName);
                         apiVersion = configuration.GetValue<string>(Constants.EdgeletApiVersionVariableName);
                         TimeSpan performanceMetricsUpdateFrequency = configuration.GetTimeSpan("PerformanceMetricsUpdateFrequency", TimeSpan.FromMinutes(5));
-                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxOpenFiles, storageLogLevel));
+                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxManifestFileSize, storageMaxOpenFiles, storageLogLevel));
                         builder.RegisterModule(new EdgeletModule(iothubHostname, deviceId, new Uri(managementUri), new Uri(workloadUri), apiVersion, dockerAuthConfig, upstreamProtocol, proxy, productInfo, closeOnIdleTimeout, idleTimeout, performanceMetricsUpdateFrequency, useServerHeartbeat, backupConfigFilePath));
                         IEnumerable<X509Certificate2> trustBundle =
                             await CertificateHelper.GetTrustBundleFromEdgelet(new Uri(workloadUri), apiVersion, Constants.WorkloadApiVersion, moduleId, moduleGenerationId);
@@ -219,7 +220,7 @@ public static async Task<int> MainAsync(IConfiguration configuration)
                             configuration.GetValue<string>(K8sConstants.EdgeK8sObjectOwnerUidKey));
                         bool runAsNonRoot = configuration.GetValue<bool>(K8sConstants.RunAsNonRootKey);
 
-                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxOpenFiles, storageLogLevel));
+                        builder.RegisterModule(new AgentModule(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.Some(new Uri(workloadUri)), Option.Some(apiVersion), moduleId, Option.Some(moduleGenerationId), enableNonPersistentStorageBackup, storageBackupPath, storageTotalMaxWalSize, storageMaxManifestFileSize, storageMaxOpenFiles, storageLogLevel));
                         builder.RegisterModule(
                             new KubernetesModule(
                                 iothubHostname,

diff --git a/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs b/edge-agent/src/Microsoft.Azure.Devices.Edge.Agent.Service/modules/AgentModule.cs
@@ -33,11 +33,38 @@ public class AgentModule : Module
         readonly bool useBackupAndRestore;
         readonly Option<string> storageBackupPath;
         readonly Option<ulong> storageTotalMaxWalSize;
+        readonly Option<ulong> storageMaxManifestFileSize;
         readonly Option<int> storageMaxOpenFiles;
         readonly Option<StorageLogLevel> storageLogLevel;
 
-        public AgentModule(int maxRestartCount, TimeSpan intensiveCareTime, int coolOffTimeUnitInSeconds, bool usePersistentStorage, string storagePath, bool useBackupAndRestore, Option<string> storageBackupPath, Option<ulong> storageTotalMaxWalSize, Option<int> storageMaxOpenFiles, Option<StorageLogLevel> storageLogLevel)
-            : this(maxRestartCount, intensiveCareTime, coolOffTimeUnitInSeconds, usePersistentStorage, storagePath, Option.None<Uri>(), Option.None<string>(), Constants.EdgeAgentModuleIdentityName, Option.None<string>(), useBackupAndRestore, storageBackupPath, storageTotalMaxWalSize, storageMaxOpenFiles, storageLogLevel)
+        public AgentModule(
+            int maxRestartCount,
+            TimeSpan intensiveCareTime,
+            int coolOffTimeUnitInSeconds,
+            bool usePersistentStorage,
+            string storagePath,
+            bool useBackupAndRestore,
+            Option<string> storageBackupPath,
+            Option<ulong> storageTotalMaxWalSize,
+            Option<ulong> storageMaxManifestFileSize,
+            Option<int> storageMaxOpenFiles,
+            Option<StorageLogLevel> storageLogLevel)
+            : this(
+                maxRestartCount,
+                intensiveCareTime,
+                coolOffTimeUnitInSeconds,
+                usePersistentStorage,
+                storagePath,
+                Option.None<Uri>(),
+                Option.None<string>(),
+                Constants.EdgeAgentModuleIdentityName,
+                Option.None<string>(),
+                useBackupAndRestore,
+                storageBackupPath,
+                storageTotalMaxWalSize,
+                storageMaxManifestFileSize,
+                storageMaxOpenFiles,
+                storageLogLevel)
         {
         }
 
@@ -54,6 +81,7 @@ public AgentModule(
             bool useBackupAndRestore,
             Option<string> storageBackupPath,
             Option<ulong> storageTotalMaxWalSize,
+            Option<ulong> storageMaxManifestFileSize,
             Option<int> storageMaxOpenFiles,
             Option<StorageLogLevel> storageLogLevel)
         {
@@ -69,6 +97,7 @@ public AgentModule(
             this.useBackupAndRestore = useBackupAndRestore;
             this.storageBackupPath = storageBackupPath;
             this.storageTotalMaxWalSize = storageTotalMaxWalSize;
+            this.storageMaxManifestFileSize = storageMaxManifestFileSize;
             this.storageMaxOpenFiles = storageMaxOpenFiles;
             this.storageLogLevel = storageLogLevel;
         }
@@ -147,7 +176,14 @@ protected override void Load(ContainerBuilder builder)
 
             // IRocksDbOptionsProvider
             // For EdgeAgent, we don't need high performance from RocksDb, so always turn off optimizeForPerformance
-            builder.Register(c => new RocksDbOptionsProvider(c.Resolve<ISystemEnvironment>(), false, this.storageTotalMaxWalSize, this.storageMaxOpenFiles, this.storageLogLevel))
+            builder
+                .Register(c => new RocksDbOptionsProvider(
+                    c.Resolve<ISystemEnvironment>(),
+                    false,
+                    this.storageTotalMaxWalSize,
+                    this.storageMaxManifestFileSize,
+                    this.storageMaxOpenFiles,
+                    this.storageLogLevel))
                 .As<IRocksDbOptionsProvider>()
                 .SingleInstance();
 

diff --git a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.Service/Constants.cs b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.Service/Constants.cs
@@ -31,6 +31,7 @@ public static class ConfigKey
             public const string EdgeHubClientCertAuthEnabled = "ClientCertAuthEnabled";
             public const string SslProtocols = "SslProtocols";
             public const string StorageMaxTotalWalSize = "RocksDB_MaxTotalWalSize";
+            public const string StorageMaxManifestFileSize = "RocksDB_MaxManifestFileSize";
             public const string StorageMaxOpenFiles = "RocksDB_MaxOpenFiles";
             public const string StorageLogLevel = "Storage_LogLevel";
             public const string ExperimentalFeatures = "experimentalFeatures";

diff --git a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.Service/DependencyManager.cs b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.Service/DependencyManager.cs
@@ -46,6 +46,7 @@ struct StoreAndForward
             public bool UseBackupAndRestore { get; }
             public Option<string> StorageBackupPath { get; }
             public Option<ulong> StorageMaxTotalWalSize { get; }
+            public Option<ulong> StorageMaxManifestFileSize { get; }
             public Option<int> StorageMaxOpenFiles { get; }
             public Option<StorageLogLevel> StorageLogLevel { get; }
 
@@ -57,6 +58,7 @@ public StoreAndForward(
                 bool useBackupAndRestore,
                 Option<string> storageBackupPath,
                 Option<ulong> storageMaxTotalWalSize,
+                Option<ulong> storageMaxManifestFileSize,
                 Option<int> storageMaxOpenFiles,
                 Option<StorageLogLevel> storageLogLevel)
             {
@@ -67,6 +69,7 @@ public StoreAndForward(
                 this.UseBackupAndRestore = useBackupAndRestore;
                 this.StorageBackupPath = storageBackupPath;
                 this.StorageMaxTotalWalSize = storageMaxTotalWalSize;
+                this.StorageMaxManifestFileSize = storageMaxManifestFileSize;
                 this.StorageMaxOpenFiles = storageMaxOpenFiles;
                 this.StorageLogLevel = storageLogLevel;
             }
@@ -315,6 +318,7 @@ void RegisterCommonModule(
                     storeAndForward.UseBackupAndRestore,
                     storeAndForward.StorageBackupPath,
                     storeAndForward.StorageMaxTotalWalSize,
+                    storeAndForward.StorageMaxManifestFileSize,
                     storeAndForward.StorageMaxOpenFiles,
                     storeAndForward.StorageLogLevel,
                     nestedEdgeEnabled));
@@ -337,6 +341,7 @@ StoreAndForward GetStoreAndForwardConfiguration()
             string storagePath = GetOrCreateDirectoryPath(this.configuration.GetValue<string>("StorageFolder"), Constants.EdgeHubStorageFolder);
             bool storeAndForwardEnabled = this.configuration.GetValue<bool>("storeAndForwardEnabled");
             Option<ulong> storageMaxTotalWalSize = this.GetConfigIfExists<ulong>(Constants.ConfigKey.StorageMaxTotalWalSize, this.configuration);
+            Option<ulong> storageMaxManifestFileSize = this.GetConfigIfExists<ulong>(Constants.ConfigKey.StorageMaxManifestFileSize, this.configuration);
             Option<int> storageMaxOpenFiles = this.GetConfigIfExists<int>(Constants.ConfigKey.StorageMaxOpenFiles, this.configuration);
             Option<StorageLogLevel> storageLogLevel = this.GetConfigIfExists<StorageLogLevel>(Constants.ConfigKey.StorageLogLevel, this.configuration);
 
@@ -354,7 +359,7 @@ StoreAndForward GetStoreAndForwardConfiguration()
             }
 
             var storeAndForwardConfiguration = new StoreAndForwardConfiguration(timeToLiveSecs);
-            return new StoreAndForward(storeAndForwardEnabled, usePersistentStorage, storeAndForwardConfiguration, storagePath, useBackupAndRestore, storageBackupPath, storageMaxTotalWalSize, storageMaxOpenFiles, storageLogLevel);
+            return new StoreAndForward(storeAndForwardEnabled, usePersistentStorage, storeAndForwardConfiguration, storagePath, useBackupAndRestore, storageBackupPath, storageMaxTotalWalSize, storageMaxManifestFileSize, storageMaxOpenFiles, storageLogLevel);
         }
 
         // TODO: Move this function to a common location that can be shared between EdgeHub and EdgeAgent

diff --git a/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.Service/modules/CommonModule.cs b/edge-hub/core/src/Microsoft.Azure.Devices.Edge.Hub.Service/modules/CommonModule.cs
@@ -49,6 +49,7 @@ public class CommonModule : Module
         readonly bool useBackupAndRestore;
         readonly Option<string> storageBackupPath;
         readonly Option<ulong> storageMaxTotalWalSize;
+        readonly Option<ulong> storageMaxManifestFileSize;
         readonly Option<int> storageMaxOpenFiles;
         readonly Option<StorageLogLevel> storageLogLevel;
         readonly bool nestedEdgeEnabled;
@@ -77,6 +78,7 @@ public CommonModule(
             bool useBackupAndRestore,
             Option<string> storageBackupPath,
             Option<ulong> storageMaxTotalWalSize,
+            Option<ulong> storageMaxManifestFileSize,
             Option<int> storageMaxOpenFiles,
             Option<StorageLogLevel> storageLogLevel,
             bool nestedEdgeEnabled)
@@ -104,6 +106,7 @@ public CommonModule(
             this.useBackupAndRestore = useBackupAndRestore;
             this.storageBackupPath = storageBackupPath;
             this.storageMaxTotalWalSize = storageMaxTotalWalSize;
+            this.storageMaxManifestFileSize = storageMaxManifestFileSize;
             this.storageMaxOpenFiles = storageMaxOpenFiles;
             this.storageLogLevel = storageLogLevel;
             this.nestedEdgeEnabled = nestedEdgeEnabled;
@@ -158,7 +161,14 @@ protected override void Load(ContainerBuilder builder)
                 .SingleInstance();
 
             // DataBase options
-            builder.Register(c => new RocksDbOptionsProvider(c.Resolve<ISystemEnvironment>(), this.optimizeForPerformance, this.storageMaxTotalWalSize, this.storageMaxOpenFiles, this.storageLogLevel))
+            builder
+                .Register(c => new RocksDbOptionsProvider(
+                    c.Resolve<ISystemEnvironment>(),
+                    this.optimizeForPerformance,
+                    this.storageMaxTotalWalSize,
+                    this.storageMaxManifestFileSize,
+                    this.storageMaxOpenFiles,
+                    this.storageLogLevel))
                 .As<IRocksDbOptionsProvider>()
                 .SingleInstance();
 

diff --git a/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/DependencyManager.cs b/edge-hub/core/test/Microsoft.Azure.Devices.Edge.Hub.E2E.Test/DependencyManager.cs
@@ -175,6 +175,7 @@ public void Register(ContainerBuilder builder)
                     enableNonPersistentStorageBackup,
                     backupFolder,
                     Option.None<ulong>(),
+                    Option.None<ulong>(),
                     Option.None<int>(),
                     Option.None<StorageLogLevel>(),
                     false));

diff --git a/edge-util/src/Microsoft.Azure.Devices.Edge.Storage.RocksDb/RocksDbOptionsProvider.cs b/edge-util/src/Microsoft.Azure.Devices.Edge.Storage.RocksDb/RocksDbOptionsProvider.cs
@@ -45,6 +45,15 @@ public class RocksDbOptionsProvider : IRocksDbOptionsProvider
         // column families whose memtables are backed by the oldest live WAL file
         const ulong DefaultMaxTotalWalSize = 512 * 1024 * 1024;
 
+        // max_manifest_file_size = 64M (bytes)
+        //
+        // The MANIFEST-<sequence number> file holds the history of operations.
+        // When the file exceeds the configured max size, a new file is created,
+        // and the previous file is marked obsolete and is deleted in the next compaction.
+        //
+        // Upstream defaults to 1G.
+        const ulong DefaultMaxManifestFileSize = 64 * 1024 * 1024;
+
         // Set a large default is to allow RocksDb to keep open,
         // but not the default unlimited setting (unlimted = -1)
         const int DefaultMaxOpenFiles = 5000;
@@ -55,14 +64,22 @@ public class RocksDbOptionsProvider : IRocksDbOptionsProvider
         readonly ISystemEnvironment env;
         readonly bool optimizeForPerformance;
         readonly ulong maxTotalWalSize;
+        readonly ulong maxManifestFileSize;
         readonly int maxOpenFiles;
         readonly StorageLogLevel logLevel;
 
-        public RocksDbOptionsProvider(ISystemEnvironment env, bool optimizeForPerformance, Option<ulong> maxTotalWalsize, Option<int> maxOpenFiles, Option<StorageLogLevel> logLevel)
+        public RocksDbOptionsProvider(
+            ISystemEnvironment env,
+            bool optimizeForPerformance,
+            Option<ulong> maxTotalWalsize,
+            Option<ulong> maxManifestFileSize,
+            Option<int> maxOpenFiles,
+            Option<StorageLogLevel> logLevel)
         {
             this.env = Preconditions.CheckNotNull(env);
             this.optimizeForPerformance = optimizeForPerformance;
             this.maxTotalWalSize = maxTotalWalsize.GetOrElse(DefaultMaxTotalWalSize);
+            this.maxManifestFileSize = maxManifestFileSize.GetOrElse(DefaultMaxManifestFileSize);
             this.maxOpenFiles = maxOpenFiles.GetOrElse(DefaultMaxOpenFiles);
             this.logLevel = logLevel.GetOrElse(DefaultLogLevel);
         }
@@ -74,6 +91,7 @@ public DbOptions GetDbOptions()
                 .SetCreateMissingColumnFamilies();
 
             options.SetMaxTotalWalSize(this.maxTotalWalSize)
+                .SetMaxManifestFileSize(this.maxManifestFileSize)
                 .SetMaxOpenFiles(this.maxOpenFiles)
                 .SetKeepLogFileNum(DefaultKeepLogFileNum)
                 .SetInfoLogLevel((int)this.logLevel);

diff --git a/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/DbStoreProviderTest.cs b/edge-util/test/Microsoft.Azure.Devices.Edge.Storage.RocksDb.Test/DbStoreProviderTest.cs
@@ -37,7 +37,13 @@ public DbStoreProviderTest()
         [Fact]
         public void CreateTest()
         {
-            var options = new RocksDbOptionsProvider(new SystemEnvironment(), true, Option.None<ulong>(), Option.None<int>(), Option.None<StorageLogLevel>());
+            var options = new RocksDbOptionsProvider(
+                new SystemEnvironment(),
+                true,
+                Option.None<ulong>(),
+                Option.None<ulong>(),
+                Option.None<int>(),
+                Option.None<StorageLogLevel>());
 
             var partitionsList1 = new[]
             {
@@ -73,7 +79,13 @@ public void CreateTest()
         [Fact]
         public async Task GetRemoveDefaultPartitionTestAsync()
         {
-            var options = new RocksDbOptionsProvider(new SystemEnvironment(), true, Option.None<ulong>(), Option.None<int>(), Option.None<StorageLogLevel>());
+            var options = new RocksDbOptionsProvider(
+                new SystemEnvironment(),
+                true,
+                Option.None<ulong>(),
+                Option.None<ulong>(),
+                Option.None<int>(),
+                Option.None<StorageLogLevel>());
 
             var partitionsList = new[]
             {