Skip to content

Commit 02729a1

Browse files
jgravelle-googlejgravelle-google
authored
First pass at LLD support for Emscripten (#1346)
* Skeleton of a beginning of o2wasm, WIP and probably not going to be used * Get building post-cherry-pick * ast->ir, remove commented out code, include a debug module print because linking * Read linking section, print emscripten metadata json * WasmBinaryWriter emits user sections on Module * Remove debugging prints, everything that isn't needed to build metadata * Rename o2wasm to lld-metadata * lld-metadata support for outputting to file * Use tables index instead of function index for initializer functions * Add lld-emscripten tool to add emscripten-runtime functions to wasm modules (built with lld) * Handle EM_ASM in lld-emscripten * Add a list of functions to forcibly export (for initializer functions) * Disable incorrect initializer function reading * Add error printing when parsing .o files in lld-metadata * Remove ';; METADATA: ' prefix from lld-metadata, output is now standalone json * Support em_asm consts that aren't at the start of a segment * Initial test framework for lld-metadata tool * Add em_asm test * Add support for WASM_INIT_FUNCS in the linking section * Remove reloc section parsing because it's unused * lld-emscripten can read and write text * Add test harness for lld-emscripten * Export all functions for now * Add missing lld test output * Add support for reading object files differently Only difference so far is in importing mutable globals being an object file representation for symbols, but invalid wasm. * Update help strings * Update linking tests for stackAlloc fix * Rename lld-emscripten,lld-metadata to wasm-emscripten-finalize,wasm-link-metadata * Add help text to header comments * auto& instead of auto & * Extract LinkType to abi/wasm-object.h * Remove special handling for wasm object file reading, allow mutable globals * Add braces around default switch case * Fix flake8 errors * Handle generating dyncall thunks for imports as well * Use explicit bool for stackPointerGlobal * Use glob patterns for lld file iteration * Use __wasm_call_ctors for all initializer functions
1 parent b01f2bb commit 02729a1

36 files changed

+1159
-62
lines changed

‎CMakeLists.txt

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,6 +275,28 @@ SET_PROPERTY(TARGET s2wasm PROPERTY CXX_STANDARD 11)
275275
SET_PROPERTY(TARGET s2wasm PROPERTY CXX_STANDARD_REQUIRED ON)
276276
INSTALL(TARGETS s2wasm DESTINATION ${CMAKE_INSTALL_BINDIR})
277277

278+
SET(wasm-link-metadata_SOURCES
279+
src/tools/wasm-link-metadata.cpp
280+
src/wasm-emscripten.cpp
281+
)
282+
ADD_EXECUTABLE(wasm-link-metadata
283+
${wasm-link-metadata_SOURCES})
284+
TARGET_LINK_LIBRARIES(wasm-link-metadata passes wasm asmjs ir cfg support)
285+
SET_PROPERTY(TARGET wasm-link-metadata PROPERTY CXX_STANDARD 11)
286+
SET_PROPERTY(TARGET wasm-link-metadata PROPERTY CXX_STANDARD_REQUIRED ON)
287+
INSTALL(TARGETS wasm-link-metadata DESTINATION ${CMAKE_INSTALL_BINDIR})
288+
289+
SET(wasm-emscripten-finalize_SOURCES
290+
src/tools/wasm-emscripten-finalize.cpp
291+
src/wasm-emscripten.cpp
292+
)
293+
ADD_EXECUTABLE(wasm-emscripten-finalize
294+
${wasm-emscripten-finalize_SOURCES})
295+
TARGET_LINK_LIBRARIES(wasm-emscripten-finalize passes wasm asmjs ir cfg support)
296+
SET_PROPERTY(TARGET wasm-emscripten-finalize PROPERTY CXX_STANDARD 11)
297+
SET_PROPERTY(TARGET wasm-emscripten-finalize PROPERTY CXX_STANDARD_REQUIRED ON)
298+
INSTALL(TARGETS wasm-emscripten-finalize DESTINATION ${CMAKE_INSTALL_BINDIR})
299+
278300
SET(wasm_as_SOURCES
279301
src/tools/wasm-as.cpp
280302
)

‎auto_update_tests.py

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,8 @@
66
from scripts.test.shared import (
77
ASM2WASM, MOZJS, NODEJS, S2WASM, WASM_SHELL, WASM_OPT, WASM_AS, WASM_DIS,
88
WASM_CTOR_EVAL, WASM_MERGE, WASM_REDUCE, WASM2ASM, WASM_METADCE,
9-
BINARYEN_INSTALL_DIR, has_shell_timeout)
9+
WASM_LINK_METADATA, WASM_EMSCRIPTEN_FINALIZE, BINARYEN_INSTALL_DIR,
10+
files_with_pattern, has_shell_timeout)
1011
from scripts.test.wasm2asm import tests, spec_tests, extra_tests, assert_tests
1112

1213
print '[ processing and updating testcases... ]\n'
@@ -72,6 +73,25 @@
7273

7374
with open(expected_file, 'w') as o: o.write(actual)
7475

76+
print '\n[ checking wasm-link-metadata testcases... ]\n'
77+
78+
for obj_path in files_with_pattern('test', 'lld', '*.o'):
79+
print '..', obj_path
80+
json_path = obj_path.replace('.o', '.json')
81+
82+
cmd = WASM_LINK_METADATA + [obj_path]
83+
actual = run_command(cmd)
84+
with open(json_path, 'w') as o: o.write(actual)
85+
86+
print '\n[ checking wasm-emscripten-finalize testcases... ]\n'
87+
88+
for wast_path in files_with_pattern('test', 'lld', '*.wast'):
89+
print '..', wast_path
90+
out_path = wast_path + '.out'
91+
cmd = WASM_EMSCRIPTEN_FINALIZE + [wast_path, '-S']
92+
actual = run_command(cmd)
93+
with open(out_path, 'w') as o: o.write(actual)
94+
7595
for t in sorted(os.listdir(os.path.join('test', 'print'))):
7696
if t.endswith('.wast'):
7797
print '..', t

‎check.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
)
3232

3333
import scripts.test.asm2wasm as asm2wasm
34+
import scripts.test.lld as lld
3435
import scripts.test.s2wasm as s2wasm
3536
import scripts.test.wasm2asm as wasm2asm
3637

@@ -604,6 +605,8 @@ def main():
604605
run_binaryen_js_tests()
605606
s2wasm.test_s2wasm()
606607
s2wasm.test_linker()
608+
lld.test_wasm_link_metadata()
609+
lld.test_wasm_emscripten_finalize()
607610
wasm2asm.test_wasm2asm()
608611
run_validator_tests()
609612
if options.torture and options.test_waterfall:

‎scripts/test/generate_lld_tests.py

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2017 WebAssembly Community Group participants
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import os
18+
import sys
19+
from support import run_command
20+
import shared
21+
22+
23+
def files_with_extensions(path, extensions):
24+
for file in sorted(os.listdir(path)):
25+
_, ext = os.path.splitext(file)
26+
if ext in extensions:
27+
yield file, ext
28+
29+
30+
def generate_object_files(clang_bin):
31+
print '\n[ building object files from C sources... ]\n'
32+
33+
lld_path = os.path.join(shared.options.binaryen_test, 'lld')
34+
for src_file, ext in files_with_extensions(lld_path, ['.c', '.cpp']):
35+
print '..', src_file
36+
obj_file = src_file.replace(ext, '.o')
37+
38+
src_path = os.path.join(lld_path, src_file)
39+
obj_path = os.path.join(lld_path, obj_file)
40+
run_command([
41+
clang_bin, src_path, '-o', obj_path,
42+
'--target=wasm32-unknown-unknown-wasm',
43+
'-c',
44+
'-nostdinc',
45+
'-Xclang', '-nobuiltininc',
46+
'-Xclang', '-nostdsysteminc',
47+
'-Xclang', '-I/s/work/emscripten/system/include',
48+
'-O1',
49+
])
50+
51+
52+
def generate_wast_files(lld_bin):
53+
print '\n[ linking wasm files from object files... ]\n'
54+
55+
lld_path = os.path.join(shared.options.binaryen_test, 'lld')
56+
for obj_file, ext in files_with_extensions(lld_path, ['.o']):
57+
print '..', obj_file
58+
wasm_file = obj_file.replace(ext, '.wasm')
59+
wast_file = obj_file.replace(ext, '.wast')
60+
61+
obj_path = os.path.join(lld_path, obj_file)
62+
wasm_path = os.path.join(lld_path, wasm_file)
63+
wast_path = os.path.join(lld_path, wast_file)
64+
run_command([
65+
lld_bin, '-flavor', 'wasm',
66+
'-z', '-stack-size=1048576',
67+
obj_path, '-o', wasm_path,
68+
'--entry=main',
69+
'--allow-undefined',
70+
'--export', '__wasm_call_ctors',
71+
])
72+
try:
73+
run_command(shared.WASM_DIS + [wasm_path, '-o', wast_path])
74+
finally:
75+
# Don't need the .wasm file, don't leave it around
76+
shared.delete_from_orbit(wasm_path)
77+
78+
79+
if __name__ == '__main__':
80+
if len(sys.argv) != 3:
81+
print 'Usage: generate_lld_tests.py [path/to/clang] [path/to/lld]'
82+
sys.exit(1)
83+
generate_object_files(sys.argv[1])
84+
generate_wast_files(sys.argv[2])

‎scripts/test/lld.py

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
#!/usr/bin/env python
2+
3+
# Copyright 2017 WebAssembly Community Group participants
4+
#
5+
# Licensed under the Apache License, Version 2.0 (the "License");
6+
# you may not use this file except in compliance with the License.
7+
# You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
import os
18+
from support import run_command
19+
from shared import (
20+
fail, fail_with_error, files_with_pattern, options,
21+
WASM_LINK_METADATA, WASM_EMSCRIPTEN_FINALIZE
22+
)
23+
24+
25+
def test_wasm_link_metadata():
26+
print '\n[ checking wasm-link-metadata testcases... ]\n'
27+
28+
for obj_path in files_with_pattern(options.binaryen_test, 'lld', '*.o'):
29+
print '..', obj_path
30+
expected_file = obj_path.replace('.o', '.json')
31+
32+
cmd = WASM_LINK_METADATA + [obj_path]
33+
actual = run_command(cmd)
34+
35+
if not os.path.exists(expected_file):
36+
print actual
37+
fail_with_error('output ' + expected_file + ' does not exist')
38+
expected = open(expected_file, 'rb').read()
39+
if actual != expected:
40+
fail(actual, expected)
41+
42+
43+
def test_wasm_emscripten_finalize():
44+
print '\n[ checking wasm-emscripten-finalize testcases... ]\n'
45+
46+
for wast_path in files_with_pattern(options.binaryen_test, 'lld', '*.wast'):
47+
print '..', wast_path
48+
expected_file = wast_path + '.out'
49+
cmd = WASM_EMSCRIPTEN_FINALIZE + [wast_path, '-S']
50+
actual = run_command(cmd)
51+
52+
if not os.path.exists(expected_file):
53+
print actual
54+
fail_with_error('output ' + expected_file + ' does not exist')
55+
expected = open(expected_file, 'rb').read()
56+
if actual != expected:
57+
fail(actual, expected)
58+
59+
60+
if __name__ == '__main__':
61+
test_wasm_link_metadata()
62+
test_wasm_emscripten_finalize()

‎scripts/test/shared.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import argparse
22
import difflib
3+
import glob
34
import os
45
import shutil
56
import subprocess
@@ -169,6 +170,9 @@ def is_exe(fpath):
169170
S2WASM = [os.path.join(options.binaryen_bin, 's2wasm')]
170171
WASM_REDUCE = [os.path.join(options.binaryen_bin, 'wasm-reduce')]
171172
WASM_METADCE = [os.path.join(options.binaryen_bin, 'wasm-metadce')]
173+
WASM_LINK_METADATA = [os.path.join(options.binaryen_bin, 'wasm-link-metadata')]
174+
WASM_EMSCRIPTEN_FINALIZE = [os.path.join(options.binaryen_bin,
175+
'wasm-emscripten-finalize')]
172176

173177
S2WASM_EXE = S2WASM[0]
174178
WASM_SHELL_EXE = WASM_SHELL[0]
@@ -429,3 +433,7 @@ def minify_check(wast, verify_final_result=True):
429433
os.unlink('a.wast')
430434
if os.path.exists('b.wast'):
431435
os.unlink('b.wast')
436+
437+
438+
def files_with_pattern(*path_pattern):
439+
return sorted(glob.glob(os.path.join(*path_pattern)))

‎src/abi/wasm-object.h

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright 2018 WebAssembly Community Group participants
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
//
18+
// Contains definitions used for wasm object files.
19+
// See: https://github.com/WebAssembly/tool-conventions/blob/master/Linking.md
20+
//
21+
22+
#ifndef wasm_abi_wasm_object_h
23+
#define wasm_abi_wasm_object_h
24+
25+
namespace wasm {
26+
27+
namespace ABI {
28+
enum LinkType : unsigned {
29+
WASM_STACK_POINTER = 0x1,
30+
WASM_SYMBOL_INFO = 0x2,
31+
WASM_DATA_SIZE = 0x3,
32+
WASM_DATA_ALIGNMENT = 0x4,
33+
WASM_SEGMENT_INFO = 0x5,
34+
WASM_INIT_FUNCS = 0x6,
35+
};
36+
} // namespace ABI
37+
38+
} // namespace wasm
39+
40+
#endif // wasm_abi_wasm_object_h

‎src/tools/s2wasm.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ int main(int argc, const char *argv[]) {
184184
std::cerr << "Emscripten gluing..." << std::endl;
185185
WasmPrinter::printModule(&wasm, std::cerr);
186186
}
187-
metadata = emscriptenGlue(
187+
metadata = ";; METADATA: " + emscriptenGlue(
188188
wasm,
189189
allowMemoryGrowth,
190190
linker.getStackPointerAddress(),

0 commit comments

Comments
 (0)