Skip to content

Commit c5bd0c3

Browse files
Running builder (content generator) functions in subprocesses on Windows
- Refactored all builder (make_*) functions into separate Python modules along to the build tree - Introduced utility function to wrap all invocations on Windows, but does not change it elsewhere - Introduced stub to use the builders module as a stand alone script and invoke a selected function There is a problem with file handles related to writing generated content (*.gen.h and *.gen.cpp) on Windows, which randomly causes a SHARING VIOLATION error to the compiler resulting in flaky builds. Running all such content generators in a new subprocess instead of directly inside the build script works around the issue. Yes, I tried the multiprocessing module. It did not work due to conflict with SCons on cPickle. Suggested workaround did not fully work either. Using the run_in_subprocess wrapper on osx and x11 platforms as well for consistency. In case of running a cross-compilation on Windows they would still be used, but likely it will not happen in practice. What counts is that the build itself is running on which platform, not the target platform. Some generated files are written directly in an SConstruct or SCsub file, before the parallel build starts. They don't need to be written in a subprocess, apparently, so I left them as is.
1 parent 92c5938 commit c5bd0c3

23 files changed

+1895
-1423
lines changed

SConstruct

+4-2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import os.path
88
import glob
99
import sys
1010
import methods
11+
import gles_builders
12+
from platform_methods import run_in_subprocess
1113

1214
# scan possible build platforms
1315

@@ -444,8 +446,8 @@ if selected_platform in platform_list:
444446
methods.no_verbose(sys, env)
445447

446448
if (not env["platform"] == "server"): # FIXME: detect GLES3
447-
env.Append( BUILDERS = { 'GLES3_GLSL' : env.Builder(action = methods.build_gles3_headers, suffix = 'glsl.gen.h', src_suffix = '.glsl') } )
448-
env.Append( BUILDERS = { 'GLES2_GLSL' : env.Builder(action = methods.build_gles2_headers, suffix = 'glsl.gen.h', src_suffix = '.glsl') } )
449+
env.Append(BUILDERS = { 'GLES3_GLSL' : env.Builder(action=run_in_subprocess(gles_builders.build_gles3_headers), suffix='glsl.gen.h', src_suffix='.glsl')})
450+
env.Append(BUILDERS = { 'GLES2_GLSL' : env.Builder(action=run_in_subprocess(gles_builders.build_gles2_headers), suffix='glsl.gen.h', src_suffix='.glsl')})
449451

450452
scons_cache_path = os.environ.get("SCONS_CACHE")
451453
if scons_cache_path != None:

core/SCsub

+25-23
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,9 @@
22

33
Import('env')
44

5-
import methods
5+
import core_builders
6+
import make_binders
7+
from platform_methods import run_in_subprocess
68

79
env.core_sources = []
810

@@ -21,7 +23,7 @@ gd_cpp += gd_inc
2123
gd_cpp += "void ProjectSettings::register_global_defaults() {\n" + gd_call + "\n}\n"
2224

2325
with open("global_defaults.gen.cpp", "w") as f:
24-
f.write(gd_cpp)
26+
f.write(gd_cpp)
2527

2628

2729
# Generate AES256 script encryption key
@@ -48,26 +50,27 @@ if ("SCRIPT_AES256_ENCRYPTION_KEY" in os.environ):
4850
txt = "0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0"
4951
print("Invalid AES256 encryption key, not 64 bits hex: " + e)
5052

53+
# NOTE: It is safe to generate this file here, since this is still executed serially
5154
with open("script_encryption_key.gen.cpp", "w") as f:
52-
f.write("#include \"project_settings.h\"\nuint8_t script_encryption_key[32]={" + txt + "};\n")
55+
f.write("#include \"project_settings.h\"\nuint8_t script_encryption_key[32]={" + txt + "};\n")
5356

5457

5558
# Add required thirdparty code. Header paths are hardcoded, we don't need to append
5659
# to the include path (saves a few chars on the compiler invocation for touchy MSVC...)
5760
thirdparty_dir = "#thirdparty/misc/"
5861
thirdparty_sources = [
59-
# C sources
60-
"base64.c",
61-
"fastlz.c",
62-
"sha256.c",
63-
"smaz.c",
64-
65-
# C++ sources
66-
"aes256.cpp",
67-
"hq2x.cpp",
68-
"md5.cpp",
69-
"pcg.cpp",
70-
"triangulator.cpp",
62+
# C sources
63+
"base64.c",
64+
"fastlz.c",
65+
"sha256.c",
66+
"smaz.c",
67+
68+
# C++ sources
69+
"aes256.cpp",
70+
"hq2x.cpp",
71+
"md5.cpp",
72+
"pcg.cpp",
73+
"triangulator.cpp",
7174
]
7275
thirdparty_sources = [thirdparty_dir + file for file in thirdparty_sources]
7376
env.add_source_files(env.core_sources, thirdparty_sources)
@@ -76,9 +79,9 @@ env.add_source_files(env.core_sources, thirdparty_sources)
7679
# However, our version has some custom modifications, so it won't compile with the system one
7780
thirdparty_minizip_dir = "#thirdparty/minizip/"
7881
thirdparty_minizip_sources = [
79-
"ioapi.c",
80-
"unzip.c",
81-
"zip.c",
82+
"ioapi.c",
83+
"unzip.c",
84+
"zip.c",
8285
]
8386
thirdparty_minizip_sources = [thirdparty_minizip_dir + file for file in thirdparty_minizip_sources]
8487
env.add_source_files(env.core_sources, thirdparty_minizip_sources)
@@ -92,20 +95,19 @@ env.add_source_files(env.core_sources, "*.cpp")
9295

9396

9497
# Make binders
95-
import make_binders
96-
env.CommandNoCache(['method_bind.gen.inc', 'method_bind_ext.gen.inc'], 'make_binders.py', make_binders.run)
98+
env.CommandNoCache(['method_bind.gen.inc', 'method_bind_ext.gen.inc'], 'make_binders.py', run_in_subprocess(make_binders.run))
9799

98100
# Authors
99101
env.Depends('#core/authors.gen.h', "../AUTHORS.md")
100-
env.CommandNoCache('#core/authors.gen.h', "../AUTHORS.md", methods.make_authors_header)
102+
env.CommandNoCache('#core/authors.gen.h', "../AUTHORS.md", run_in_subprocess(core_builders.make_authors_header))
101103

102104
# Donors
103105
env.Depends('#core/donors.gen.h', "../DONORS.md")
104-
env.CommandNoCache('#core/donors.gen.h', "../DONORS.md", methods.make_donors_header)
106+
env.CommandNoCache('#core/donors.gen.h', "../DONORS.md", run_in_subprocess(core_builders.make_donors_header))
105107

106108
# License
107109
env.Depends('#core/license.gen.h', ["../COPYRIGHT.txt", "../LICENSE.txt"])
108-
env.CommandNoCache('#core/license.gen.h', ["../COPYRIGHT.txt", "../LICENSE.txt"], methods.make_license_header)
110+
env.CommandNoCache('#core/license.gen.h', ["../COPYRIGHT.txt", "../LICENSE.txt"], run_in_subprocess(core_builders.make_license_header))
109111

110112
# Chain load SCsubs
111113
SConscript('os/SCsub')

core/core_builders.py

+236
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,236 @@
1+
"""Functions used to generate source files during build time
2+
3+
All such functions are invoked in a subprocess on Windows to prevent build flakiness.
4+
5+
"""
6+
from platform_methods import subprocess_main
7+
from compat import iteritems, itervalues, open_utf8, escape_string
8+
9+
10+
def make_authors_header(target, source, env):
11+
sections = ["Project Founders", "Lead Developer", "Project Manager", "Developers"]
12+
sections_id = ["AUTHORS_FOUNDERS", "AUTHORS_LEAD_DEVELOPERS", "AUTHORS_PROJECT_MANAGERS", "AUTHORS_DEVELOPERS"]
13+
14+
src = source[0]
15+
dst = target[0]
16+
f = open_utf8(src, "r")
17+
g = open_utf8(dst, "w")
18+
19+
g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
20+
g.write("#ifndef _EDITOR_AUTHORS_H\n")
21+
g.write("#define _EDITOR_AUTHORS_H\n")
22+
23+
reading = False
24+
25+
def close_section():
26+
g.write("\t0\n")
27+
g.write("};\n")
28+
29+
for line in f:
30+
if reading:
31+
if line.startswith(" "):
32+
g.write("\t\"" + escape_string(line.strip()) + "\",\n")
33+
continue
34+
if line.startswith("## "):
35+
if reading:
36+
close_section()
37+
reading = False
38+
for section, section_id in zip(sections, sections_id):
39+
if line.strip().endswith(section):
40+
current_section = escape_string(section_id)
41+
reading = True
42+
g.write("const char *const " + current_section + "[] = {\n")
43+
break
44+
45+
if reading:
46+
close_section()
47+
48+
g.write("#endif\n")
49+
50+
g.close()
51+
f.close()
52+
53+
54+
def make_donors_header(target, source, env):
55+
sections = ["Platinum sponsors", "Gold sponsors", "Mini sponsors",
56+
"Gold donors", "Silver donors", "Bronze donors"]
57+
sections_id = ["DONORS_SPONSOR_PLAT", "DONORS_SPONSOR_GOLD", "DONORS_SPONSOR_MINI",
58+
"DONORS_GOLD", "DONORS_SILVER", "DONORS_BRONZE"]
59+
60+
src = source[0]
61+
dst = target[0]
62+
f = open_utf8(src, "r")
63+
g = open_utf8(dst, "w")
64+
65+
g.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
66+
g.write("#ifndef _EDITOR_DONORS_H\n")
67+
g.write("#define _EDITOR_DONORS_H\n")
68+
69+
reading = False
70+
71+
def close_section():
72+
g.write("\t0\n")
73+
g.write("};\n")
74+
75+
for line in f:
76+
if reading >= 0:
77+
if line.startswith(" "):
78+
g.write("\t\"" + escape_string(line.strip()) + "\",\n")
79+
continue
80+
if line.startswith("## "):
81+
if reading:
82+
close_section()
83+
reading = False
84+
for section, section_id in zip(sections, sections_id):
85+
if line.strip().endswith(section):
86+
current_section = escape_string(section_id)
87+
reading = True
88+
g.write("const char *const " + current_section + "[] = {\n")
89+
break
90+
91+
if reading:
92+
close_section()
93+
94+
g.write("#endif\n")
95+
96+
g.close()
97+
f.close()
98+
99+
100+
def make_license_header(target, source, env):
101+
src_copyright = source[0]
102+
src_license = source[1]
103+
dst = target[0]
104+
105+
class LicenseReader:
106+
def __init__(self, license_file):
107+
self._license_file = license_file
108+
self.line_num = 0
109+
self.current = self.next_line()
110+
111+
def next_line(self):
112+
line = self._license_file.readline()
113+
self.line_num += 1
114+
while line.startswith("#"):
115+
line = self._license_file.readline()
116+
self.line_num += 1
117+
self.current = line
118+
return line
119+
120+
def next_tag(self):
121+
if not ':' in self.current:
122+
return ('', [])
123+
tag, line = self.current.split(":", 1)
124+
lines = [line.strip()]
125+
while self.next_line() and self.current.startswith(" "):
126+
lines.append(self.current.strip())
127+
return (tag, lines)
128+
129+
from collections import OrderedDict
130+
projects = OrderedDict()
131+
license_list = []
132+
133+
with open_utf8(src_copyright, "r") as copyright_file:
134+
reader = LicenseReader(copyright_file)
135+
part = {}
136+
while reader.current:
137+
tag, content = reader.next_tag()
138+
if tag in ("Files", "Copyright", "License"):
139+
part[tag] = content[:]
140+
elif tag == "Comment":
141+
# attach part to named project
142+
projects[content[0]] = projects.get(content[0], []) + [part]
143+
144+
if not tag or not reader.current:
145+
# end of a paragraph start a new part
146+
if "License" in part and not "Files" in part:
147+
# no Files tag in this one, so assume standalone license
148+
license_list.append(part["License"])
149+
part = {}
150+
reader.next_line()
151+
152+
data_list = []
153+
for project in itervalues(projects):
154+
for part in project:
155+
part["file_index"] = len(data_list)
156+
data_list += part["Files"]
157+
part["copyright_index"] = len(data_list)
158+
data_list += part["Copyright"]
159+
160+
with open_utf8(dst, "w") as f:
161+
162+
f.write("/* THIS FILE IS GENERATED DO NOT EDIT */\n")
163+
f.write("#ifndef _EDITOR_LICENSE_H\n")
164+
f.write("#define _EDITOR_LICENSE_H\n")
165+
f.write("const char *const GODOT_LICENSE_TEXT =")
166+
167+
with open_utf8(src_license, "r") as license_file:
168+
for line in license_file:
169+
escaped_string = escape_string(line.strip())
170+
f.write("\n\t\t\"" + escaped_string + "\\n\"")
171+
f.write(";\n\n")
172+
173+
f.write("struct ComponentCopyrightPart {\n"
174+
"\tconst char *license;\n"
175+
"\tconst char *const *files;\n"
176+
"\tconst char *const *copyright_statements;\n"
177+
"\tint file_count;\n"
178+
"\tint copyright_count;\n"
179+
"};\n\n")
180+
181+
f.write("struct ComponentCopyright {\n"
182+
"\tconst char *name;\n"
183+
"\tconst ComponentCopyrightPart *parts;\n"
184+
"\tint part_count;\n"
185+
"};\n\n")
186+
187+
f.write("const char *const COPYRIGHT_INFO_DATA[] = {\n")
188+
for line in data_list:
189+
f.write("\t\"" + escape_string(line) + "\",\n")
190+
f.write("};\n\n")
191+
192+
f.write("const ComponentCopyrightPart COPYRIGHT_PROJECT_PARTS[] = {\n")
193+
part_index = 0
194+
part_indexes = {}
195+
for project_name, project in iteritems(projects):
196+
part_indexes[project_name] = part_index
197+
for part in project:
198+
f.write("\t{ \"" + escape_string(part["License"][0]) + "\", "
199+
+ "&COPYRIGHT_INFO_DATA[" + str(part["file_index"]) + "], "
200+
+ "&COPYRIGHT_INFO_DATA[" + str(part["copyright_index"]) + "], "
201+
+ str(len(part["Files"])) + ", "
202+
+ str(len(part["Copyright"])) + " },\n")
203+
part_index += 1
204+
f.write("};\n\n")
205+
206+
f.write("const int COPYRIGHT_INFO_COUNT = " + str(len(projects)) + ";\n")
207+
208+
f.write("const ComponentCopyright COPYRIGHT_INFO[] = {\n")
209+
for project_name, project in iteritems(projects):
210+
f.write("\t{ \"" + escape_string(project_name) + "\", "
211+
+ "&COPYRIGHT_PROJECT_PARTS[" + str(part_indexes[project_name]) + "], "
212+
+ str(len(project)) + " },\n")
213+
f.write("};\n\n")
214+
215+
f.write("const int LICENSE_COUNT = " + str(len(license_list)) + ";\n")
216+
217+
f.write("const char *const LICENSE_NAMES[] = {\n")
218+
for l in license_list:
219+
f.write("\t\"" + escape_string(l[0]) + "\",\n")
220+
f.write("};\n\n")
221+
222+
f.write("const char *const LICENSE_BODIES[] = {\n\n")
223+
for l in license_list:
224+
for line in l[1:]:
225+
if line == ".":
226+
f.write("\t\"\\n\"\n")
227+
else:
228+
f.write("\t\"" + escape_string(line) + "\\n\"\n")
229+
f.write("\t\"\",\n\n")
230+
f.write("};\n\n")
231+
232+
f.write("#endif\n")
233+
234+
235+
if __name__ == '__main__':
236+
subprocess_main(globals())

core/make_binders.py

+7-2
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
# -*- coding: ibm850 -*-
2+
from platform_methods import subprocess_main
23

34

45
template_typed = """
@@ -265,8 +266,12 @@ def run(target, source, env):
265266
else:
266267
text += t
267268

268-
with open(target[0].path, "w") as f:
269+
with open(target[0], "w") as f:
269270
f.write(text)
270271

271-
with open(target[1].path, "w") as f:
272+
with open(target[1], "w") as f:
272273
f.write(text_ext)
274+
275+
276+
if __name__ == '__main__':
277+
subprocess_main(globals())

0 commit comments

Comments
 (0)