Merge branch 'fastbuild-concurrency-group' into 'main'

Implement support for Fastbuild Concurrency Groups

See merge request Sharpmake/sharpmake!595

Sharpmake.Generators/FastBuild/Bff.Template.cs

@@ -53,8 +53,18 @@ public static class ConfigurationFile
     [CachePath]
     [WorkerConnectionLimit]
     .AllowDBMigration_Experimental = [fastBuildAllowDBMigration]
+    .ConcurrencyGroups = [fastbuildConcurrencyGroupList]
 [AdditionalGlobalSettings]
 }
+";
+
+                public static string ConcurrencyGroup =
+@"[fastBuildConcurrencyGroupSectionName] =
+[
+    .ConcurrencyGroupName = '[fastBuildConcurrencyGroupName]'
+    .ConcurrencyLimit = [fastBuildConcurrencyLimit]
+    .ConcurrencyPerJobMiB = [fastBuildConcurrencyPerJobMiB]
+]
 ";
 
                 public const string WinEnvironment =
@@ -520,6 +530,7 @@ public static class ConfigurationFile
     .LinkerLinkObjects      = [fastBuildLinkerLinkObjects]
     .LinkerStampExe         = [fastBuildStampExecutable]
     .LinkerStampExeArgs     = [fastBuildStampArguments]
+    .ConcurrencyGroupName   = '[fastbuildConcurrencyGroupName]'
 ";
 
                 public static string ResourcesBeginSection = @"

Sharpmake.Generators/FastBuild/Bff.cs

@@ -1187,6 +1187,7 @@ List<string> skipFiles
                                     using (bffGenerator.Declare("fastBuildStampExecutable", fastBuildStampExecutable))
                                     using (bffGenerator.Declare("fastBuildStampArguments", fastBuildStampArguments))
                                     using (bffGenerator.Declare("fastBuildEmbeddedOutputPrefix", fastBuildEmbeddedOutputPrefix))
+                                    using (bffGenerator.Declare("fastbuildConcurrencyGroupName", conf.FastBuildLinkConcurrencyGroup ?? FileGeneratorUtilities.RemoveLineTag))
                                     {
                                         if (projectHasResourceFiles)
                                         {

Sharpmake.Generators/FastBuild/MasterBff.cs

@@ -664,6 +664,34 @@ private static void WriteMasterSettingsSection(FileGenerator masterBffGenerator,
             }
 
             using (masterBffGenerator.Declare("fastBuildProjectName", "Master"))
+            {
+                masterBffGenerator.Write(Bff.Template.ConfigurationFile.HeaderFile);
+            }
+
+            string concurrencyGroupList = FileGeneratorUtilities.RemoveLineTag;
+            if (FastBuildSettings.ConcurrencyGroups.Count > 0)
+            {
+                masterBffGenerator.WriteLine("//------------------------------");
+                masterBffGenerator.WriteLine("// Concurrency groups definition");
+                masterBffGenerator.WriteLine("//------------------------------");
+                List<string> groupSectionList = new List<string>();
+
+                foreach (var group in FastBuildSettings.ConcurrencyGroups)
+                {
+                    string groupSectionName = $".ConcurrencyGroup{group.Key}";
+                    groupSectionList.Add(groupSectionName); 
+
+                    using (masterBffGenerator.Declare("fastBuildConcurrencyGroupName", group.Key))
+                    using (masterBffGenerator.Declare("fastBuildConcurrencyGroupSectionName", groupSectionName))
+                    using (masterBffGenerator.Declare("fastBuildConcurrencyLimit", group.Value.ConcurrencyLimit.HasValue ? group.Value.ConcurrencyLimit.ToString() : FileGeneratorUtilities.RemoveLineTag))
+                    using (masterBffGenerator.Declare("fastBuildConcurrencyPerJobMiB", group.Value.ConcurrencyPerJobMiB.HasValue ? group.Value.ConcurrencyPerJobMiB : FileGeneratorUtilities.RemoveLineTag))
+                    {
+                        masterBffGenerator.Write(Bff.Template.ConfigurationFile.ConcurrencyGroup);
+                    }
+                }
+                concurrencyGroupList = UtilityMethods.FBuildFormatList(groupSectionList, 4, UtilityMethods.FBuildFormatListOptions.UseSingleElementShortFormat | UtilityMethods.FBuildFormatListOptions.UseCommaBetweenElements);
+            }
+
             using (masterBffGenerator.Declare("CachePath", cachePath))
             using (masterBffGenerator.Declare("CachePluginDLL", cachePluginDLL))
             using (masterBffGenerator.Declare("WorkerConnectionLimit", workerConnectionLimit))
@@ -674,8 +702,8 @@ private static void WriteMasterSettingsSection(FileGenerator masterBffGenerator,
             using (masterBffGenerator.Declare("fastBuildEnvironments", fastBuildEnvironments))
             using (masterBffGenerator.Declare("envRemoveGuards", envRemoveGuards))
             using (masterBffGenerator.Declare("envAdditionalVariables", envAdditionalVariables))
+            using (masterBffGenerator.Declare("fastbuildConcurrencyGroupList", concurrencyGroupList))
             {
-                masterBffGenerator.Write(Bff.Template.ConfigurationFile.HeaderFile);
                 masterBffGenerator.Write(Bff.Template.ConfigurationFile.GlobalSettings);
             }
         }

Sharpmake/FastBuildSettings.cs

@@ -208,5 +208,50 @@ public enum CacheTypes
         /// Custom arguments pass to fastbuild
         /// </summary>
         public static string FastBuildCustomArguments = null;
+
+        /// <summary>
+        /// Enable the use of Fastbuild concurrency groups. You can only enable it if your fastbuild version supports this feature.
+        /// </summary>
+        public static bool EnableConcurrencyGroups { get; set; } = false;
+
+        /// <summary>
+        /// This struct is used to define concurrency groups. See fastbuild documentation for more details
+        /// </summary>
+        public struct ConcurrencyGroup
+        {
+            public int? ConcurrencyLimit; // Max number of concurrent job for this group
+            public int? ConcurrencyPerJobMiB; // Arbitrary limit of memory per job in MiB. This is used to limit the number of concurrent jobs based on memory usage.
+        }
+
+        private static Dictionary<string, ConcurrencyGroup> _concurrencyGroups = new Dictionary<string, ConcurrencyGroup>();
+
+        /// <summary>
+        /// List of concurrency groups used by sharpmake when defining the fastbuild configurations. 
+        /// Concurrency groups can be used to limit the number of parallel processes using the same concurrency group. See fastbuild documentation for more details.
+        /// It is an optional feature and will only be used when EnableConcurrencyGroups is set to true.
+        /// </summary>
+        public static IReadOnlyDictionary<string, ConcurrencyGroup> ConcurrencyGroups = _concurrencyGroups;
+
+        /// <summary>
+        /// Add a concurrency group to the list of concurrency groups.
+        /// </summary>
+        /// <param name="groupName">concurrency group name</param>
+        /// <param name="group">group params</param>
+        /// <exception cref="Error"></exception>
+        public static void AddConcurrencyGroup(string groupName, ConcurrencyGroup group)
+        {
+            // Validate the group name... We use the group name to build the concurrency struct identifier so it has to be a valid identifier.
+            if (!System.Text.RegularExpressions.Regex.IsMatch(groupName, "^[a-zA-Z0-9_\\-]+$"))
+            {
+                throw new Error($"Fastbuild concurrency group name must be a valid identifier. Name: {groupName}");
+            }
+
+            if (!group.ConcurrencyLimit.HasValue && !group.ConcurrencyPerJobMiB.HasValue)
+            {
+                throw new Error($"Concurrency group must have at least one of ConcurrencyLimit or ConcurrencyPerJobMiB set. Group: {groupName}");
+            }
+
+            _concurrencyGroups.Add(groupName, group);
+        }
     }
 }

Sharpmake/Project.Configuration.cs

@@ -1683,6 +1683,28 @@ public enum FastBuildClangMscVersionDetectionType
             /// </summary>
             public FastBuildClangMscVersionDetectionType FastBuildClangMscVersionDetectionInfo = FastBuildClangMscVersionDetectionType.FullVersion;
 
+            private string _fastBuildLinkConcurrencyGroup = null;
+            /// <summary>
+            /// Optional fastbuild concurrency group name. Concurrency groups are used to limit the number of parallel processes using the same concurrency group.
+            /// It can be used for example to limit the number of LTO link process to 1.
+            /// </summary>
+            public string FastBuildLinkConcurrencyGroup
+            {
+                get
+                {
+                    return _fastBuildLinkConcurrencyGroup;
+                }
+                set
+                {
+                    if (!FastBuildSettings.EnableConcurrencyGroups)
+                        throw new Error("Can't set FastBuildLinkConcurrencyGroup as FastBuildSettings.EnableConcurrencyGroups is false");
+                    if (!FastBuildSettings.ConcurrencyGroups.ContainsKey(value))
+                        throw new Error($"Can't set FastBuildLinkConcurrencyGroup to {value} as it is not defined in FastBuildSettings.ConcurrencyGroups");
+
+                    _fastBuildLinkConcurrencyGroup = value;
+                }
+            }
+
             private Strings _intellisenseAdditionalDefines;
 
             /// <summary>

samples/FastBuildSimpleExecutable/FastBuildSimpleExecutable.sharpmake.cs

@@ -53,6 +53,7 @@ public void ConfigureFastBuild(Configuration conf, Target target)
 
             conf.IntellisenseAdditionalDefines.Add("MY_INTELLISENSE_DEFINE", "MY_INTELLISENSE_DEFINE2");
             conf.IntellisenseAdditionalCommandLineOptions.Add("/MY_INTELLISENSE_OPTION", "/MY_INTELLISENSE_OPTION2"); // Dummy options just to validate the output
+            conf.FastBuildLinkConcurrencyGroup = "Test1";
         }
 
         [Configure(Optimization.Release)]
@@ -120,6 +121,13 @@ public static void SharpmakeMain(Sharpmake.Arguments arguments)
             KitsRootPaths.SetUseKitsRootForDevEnv(DevEnv.vs2019, KitsRootEnum.KitsRoot10, Options.Vc.General.WindowsTargetPlatformVersion.v10_0_19041_0);
             KitsRootPaths.SetUseKitsRootForDevEnv(DevEnv.vs2022, KitsRootEnum.KitsRoot10, Options.Vc.General.WindowsTargetPlatformVersion.v10_0_19041_0);
 
+            // Defining several groups to insure that the generation is correct but only the first one is truly used.
+            // Compiling debug + release at same time while -monitor is specified on commandline will show that a single compilation is done at a time(see %TEMP%\FastBuildLog.Log).
+            FastBuildSettings.EnableConcurrencyGroups = true;
+            FastBuildSettings.AddConcurrencyGroup("Test1", new FastBuildSettings.ConcurrencyGroup { ConcurrencyLimit = 1 });
+            FastBuildSettings.AddConcurrencyGroup("Test2", new FastBuildSettings.ConcurrencyGroup { ConcurrencyPerJobMiB = 5000 });
+            FastBuildSettings.AddConcurrencyGroup("Test3", new FastBuildSettings.ConcurrencyGroup { ConcurrencyLimit = 42, ConcurrencyPerJobMiB = 5000 });
+
             arguments.Generate<FastBuildSolution>();
         }
     }