Add option to use stevedore or ono for external build deps

.gitlab-ci.yml

@@ -18,15 +18,26 @@ run_all_builds:
 build_osx_runtime:
   stage: build
   tags:
-    - buildfarm
-    - darwin
+    - bokken-job
+  variables:
+    BOKKEN_VM: build_osx_runtime_vm
+    BOKKEN_JOB: |
+      resources:
+        - name: build_osx_runtime_vm
+          image: buildfarm/mac:latest
+          flavor: m1.mac
+          type: Unity::VM::osx
+          num_instances: 1
+          config:
+            env_vars:
+              - PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin
   script:
   - git submodule update --init --recursive
   - chmod +x external/buildscripts/bee
   - cd external/buildscripts
   - ./bee
   - cd ../..
-  - perl external/buildscripts/build_runtime_osx.pl
+  - perl external/buildscripts/build_runtime_osx.pl --stevedorebuilddeps=1
   - mkdir -p incomingbuilds/osx-i386
   - cp -r builds/ incomingbuilds/osx-i386/
   artifacts:
@@ -40,15 +51,26 @@ build_osx_runtime:
 build_osx_classlibs:
   stage: build
   tags:
-    - buildfarm
-    - darwin
+    - bokken-job
+  variables:
+    BOKKEN_VM: build_osx_classlibs_vm
+    BOKKEN_JOB: |
+      resources:
+        - name: build_osx_classlibs_vm
+          image: buildfarm/mac:latest
+          flavor: m1.mac
+          type: Unity::VM::osx
+          num_instances: 1
+          config:
+            env_vars:
+              - PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin
   script:
   - git submodule update --init --recursive
   - chmod +x external/buildscripts/bee
   - cd external/buildscripts
   - ./bee
   - cd ../..
-  - perl external/buildscripts/build_classlibs_osx.pl
+  - perl external/buildscripts/build_classlibs_osx.pl --stevedorebuilddeps=1
   - mkdir -p incomingbuilds/classlibs
   - cp -r ZippedClasslibs.tar.gz incomingbuilds/classlibs/
   - cd incomingbuilds/classlibs
@@ -66,15 +88,26 @@ build_osx_classlibs:
 build_android:
   stage: build
   tags:
-    - buildfarm
-    - darwin
+    - bokken-job
+  variables:
+    BOKKEN_VM: build_android_vm
+    BOKKEN_JOB: |
+      resources:
+        - name: build_android_vm
+          image: buildfarm/mac:latest
+          flavor: m1.mac
+          type: Unity::VM::osx
+          num_instances: 1
+          config:
+            env_vars:
+              - PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin
   script:
   - git submodule update --init --recursive
   - chmod +x external/buildscripts/bee
   - cd external/buildscripts
   - ./bee
   - cd ../..
-  - bash external/buildscripts/build_runtime_android.sh
+  - perl external/buildscripts/build_runtime_android.pl --stevedorebuilddeps=1
   - mkdir -p incomingbuilds/android/
   - cp -r builds/* incomingbuilds/android/
   artifacts:
@@ -95,7 +128,7 @@ build_win:
   - cd external/buildscripts
   - ./bee.exe
   - cd ../..
-  - perl external/buildscripts/build_runtime_win64.pl
+  - perl external/buildscripts/build_runtime_win64.pl --stevedorebuilddeps=1
   - mkdir -p incomingbuilds/win64
   - cp -r builds/* incomingbuilds/win64/
   artifacts:
@@ -116,7 +149,7 @@ build_win_x86:
   - cd external/buildscripts
   - ./bee.exe
   - cd ../..
-  - perl external/buildscripts/build_runtime_win.pl
+  - perl external/buildscripts/build_runtime_win.pl --stevedorebuilddeps=1
   - mkdir -p incomingbuilds/win32
   - cp -r builds/* incomingbuilds/win32/
   artifacts:
@@ -159,7 +192,7 @@ build_linux_x64:
   - cd external/buildscripts
   - ./bee
   - cd ../..
-  - perl external/buildscripts/build_runtime_linux.pl -build64=1
+  - perl external/buildscripts/build_runtime_linux.pl -build64=1 --stevedorebuilddeps=1
   - mkdir -p incomingbuilds/linux64
   - cp -r builds/* incomingbuilds/linux64/
   artifacts:
@@ -181,7 +214,7 @@ build_linux_x86:
   - cd external/buildscripts
   - ./bee
   - cd ../..
-  - perl external/buildscripts/build_runtime_linux.pl
+  - perl external/buildscripts/build_runtime_linux.pl --stevedorebuilddeps=1
   - mkdir -p incomingbuilds/linux32
   - cp -r builds/* incomingbuilds/linux32/
   artifacts:

external/buildscripts/Build.bee.cs

@@ -23,9 +23,6 @@ internal static void Main()
 			var buildScriptsRoot = monoRoot.Combine("external").Combine("buildscripts");
 			Console.WriteLine(">>> Build scripts directory: " + buildScriptsRoot);
 
-			var monoBuildDeps = monoRoot.Parent.Parent.Combine("mono-build-deps").Combine("build");
-			Console.WriteLine(">>> Mono build dependecies directory: " + monoBuildDeps);
-
 			var buildDependenciesConfigFile = buildScriptsRoot.Combine("buildDependencies.txt");
 			Console.WriteLine(">>> Mono build dependecies stevedore version config file: " + buildDependenciesConfigFile);
 
@@ -34,21 +31,14 @@ internal static void Main()
 
 			if (buildDependenciesConfigFile.Exists())
 			{
-				if (!monoBuildDeps.DirectoryExists())
-				{
-					Console.WriteLine(">>> " + monoBuildDeps + " does not exist. Creating it ...");
-					monoBuildDeps.CreateDirectory();
-				}
+				var artifactList = ParseBuildDependenciesConfigFile(buildDependenciesConfigFile.ToString());
 
-				var artifactNameIdFilesDictionary = ParseBuildDependenciesConfigFile(buildDependenciesConfigFile.ToString());
-
-				foreach (var item in artifactNameIdFilesDictionary)
+				foreach (var item in artifactList)
 				{
-					var artifactName = item.Key.Item1;
-					var artifactId = item.Key.Item2;
-					var repoName = item.Key.Item3;
-					var artifactFiles = item.Value;
-					DownloadAndCopyArtifact(artifactId, artifactName, repoName, artifactFiles, monoBuildDeps, stevedoreArtifactsDir);
+					var artifactName = item.Item1;
+					var artifactId = item.Item2;
+					var repoName = item.Item3;
+					DownloadArtifact(artifactId, artifactName, repoName);
 				}
 			}
 			else
@@ -57,46 +47,13 @@ internal static void Main()
 			}
 		}
 
-		private static void DownloadAndCopyArtifact(string artifactId, string artifactName, string repoName, IEnumerable<NPath> artifacts, NPath monoBuildDeps, NPath stevedoreArtifactsDir)
+		private static void DownloadArtifact(string artifactId, string artifactName, string repoName)
 		{
+			Console.WriteLine($">>> Registering artifact {artifactName}");
 			var artifact = new StevedoreArtifact(repoName, new ArtifactId(artifactId));
 			Backend.Current.Register(artifact);
-
-			var inputs = new List<NPath>();
-			var targetFiles = new List<NPath>();
-			foreach (var item in artifacts)
-			{
-				inputs.Add(stevedoreArtifactsDir.Combine(artifactName).Combine(item));
-				targetFiles.Add(monoBuildDeps.Combine(artifactName).Combine(item));
-			}
-
-			var targetDir = monoBuildDeps;
-			if (HostPlatform.IsWindows)
-			{
-				targetDir = monoBuildDeps.Combine(artifactName);
-			}
-
-			Backend.Current.AddAction(
-				actionName: "CopyArtifact",
-				targetFiles: targetFiles.ToArray(),
-				inputs: inputs.ToArray(),
-				executableStringFor: ExecutableStringForDirectoryCopy(stevedoreArtifactsDir.Combine(artifactName), targetDir),
-				commandLineArguments: new string[] { },
-				allowUnwrittenOutputFiles: true
-			);
 		}
 
-		private static void ExecuteBuildScript(NPath[] inputFiles, NPath buildScript, NPath buildRoot)
-		{
-			Backend.Current.AddAction(
-				actionName: "ExecuteBuildScript",
-				targetFiles: new[] { buildRoot},
-				inputs: inputFiles,
-				executableStringFor: $"perl {buildScript}",
-				commandLineArguments: new string[] { },
-				allowUnwrittenOutputFiles: true
-			);
-		}
 		private static NPath GetMonoRootDir()
 		{
 			var exePath = new NPath(System.Reflection.Assembly.GetEntryAssembly().Location);
@@ -109,13 +66,6 @@ private static NPath GetMonoRootDir()
 			return monoRoot;
 		}
 
-		private static string ExecutableStringForDirectoryCopy(NPath from, NPath target)
-		{
-			return HostPlatform.IsWindows
-				? $"xcopy {from.InQuotes(SlashMode.Native)} {target.InQuotes(SlashMode.Native)} /s /e /d /Y"
-				: $"cp -r -v {from.InQuotes(SlashMode.Native)} {target.InQuotes(SlashMode.Native)}";
-		}
-
 		private static bool IsRunningOnBuildMachine()
 		{
 			var buildMachine = Environment.GetEnvironmentVariable("UNITY_THISISABUILDMACHINE");
@@ -128,21 +78,18 @@ private static bool IsRunningOnBuildMachine()
 			# name : <stevedore artifact name>
 			# id : <stevedore artifact id>
 			# repo : <stevedore repo name (can be testing/public/unityinternal)> 
-			# files : <folder and/or comma-separated list of files downloaded and unpacked> 
 
 			name: 7z
 			id: 7z/9df1e3b3b120_12ed325f6a47f0e5cebc247dbe9282a5da280d392cce4e6c9ed227d57ff1e2ff.7z
 			repo: testing
-			files : 7z
 
 			name: libgdiplus
 			id : libgdiplus/9df1e3b3b120_4cf7c08770db93922f54f38d2461b9122cddc898db58585864446e70c5ad3057.7z
 			repo: public
-			files : libgdiplus,lib2
 		*/
-		private static Dictionary<Tuple<string, string, string>, List<NPath>> ParseBuildDependenciesConfigFile(string buildDependenciesConfigFile)
+		private static List<Tuple<string, string, string>> ParseBuildDependenciesConfigFile(string buildDependenciesConfigFile)
 		{
-			var artifactNameIdFilesDictionary = new Dictionary<Tuple<string, string, string>, List<NPath>>();
+			var artifactNameIdFilesDictionary = new List<Tuple<string, string, string>>();
 
 			var fileStream = new FileStream(buildDependenciesConfigFile, FileMode.Open, FileAccess.Read);
 			using (var streamReader = new StreamReader(fileStream, Encoding.UTF8))
@@ -158,7 +105,6 @@ private static Dictionary<Tuple<string, string, string>, List<NPath>> ParseBuild
 							var name = "";
 							var id = "";
 							var repoName = "";
-							var files = "";
 
 							//read name
 							name = line.Split(':')[1].Trim();
@@ -175,25 +121,7 @@ private static Dictionary<Tuple<string, string, string>, List<NPath>> ParseBuild
 							else
 								throw new Exception($">>> Invalid {buildDependenciesConfigFile}, repo name does not exist");
 
-							//read comma separated folder/files list
-							if ((line = streamReader.ReadLine()) != null)
-								files = line.Split(':')[1].Trim();
-							else
-								throw new Exception($">>> Invalid {buildDependenciesConfigFile}, files do not exist");
-
-							var filesList = new List<NPath>();
-							if (!string.IsNullOrEmpty(files))
-							{
-								if (files.Contains(","))
-									files.Split(',').ForEach(f => { filesList.Add(new NPath(f.Trim())); });
-								else
-									filesList.Add(new NPath(files.Trim()));
-							}
-							else
-							{
-								throw new Exception($">>> Invalid {buildDependenciesConfigFile}");
-							}
-							artifactNameIdFilesDictionary.Add(new Tuple<string, string, string>(name, id, repoName), filesList);
+							artifactNameIdFilesDictionary.Add(new Tuple<string, string, string>(name, id, repoName));
 						}
 					}
 				}