Skip to content

Commit fdd4ee2

Browse files
author
Jianchun Xu
committed
xplat: enhance test/runtests.py to use rlexe.xml
Use `argparse` module to handle command line arguments. (Now this also works on Windows -- needs **python 2.7**.) Read `rlexe.xml` to load test metadata and honor `compile-flags`. Run all loaded tests and print summary at the end. Note this is only one step towards current rl test harness. There are lots of test harness features not implemented yet.
1 parent 2740f5e commit fdd4ee2

File tree

1 file changed

+139
-82
lines changed

1 file changed

+139
-82
lines changed

‎test/runtests.py‎

Lines changed: 139 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -7,43 +7,64 @@
77
import sys
88
import os
99
import subprocess as SP
10+
import argparse, textwrap
11+
import xml.etree.ElementTree as ET
12+
13+
parser = argparse.ArgumentParser(
14+
description='ChakraCore *nix Test Script',
15+
formatter_class=argparse.RawDescriptionHelpFormatter,
16+
epilog=textwrap.dedent('''\
17+
Samples:
18+
19+
test all folders:
20+
{0}
21+
22+
test only Array:
23+
{0} Array
24+
25+
test a single file:
26+
{0} Basics/hello.js
27+
'''.format(sys.argv[0]))
28+
)
29+
parser.add_argument('folders', metavar='folder', nargs='*',
30+
help='folder subset to run tests')
31+
parser.add_argument('-b', '--binary', metavar='binary', help='ch full path');
32+
parser.add_argument('-d', '--debug', action='store_true',
33+
help='use debug build');
34+
parser.add_argument('-t', '--test', action='store_true', help='use test build');
35+
parser.add_argument('--x86', action='store_true', help='use x86 build');
36+
parser.add_argument('--x64', action='store_true', help='use x64 build');
37+
args = parser.parse_args()
1038

11-
test_all = True
12-
test_root = os.path.dirname(os.path.realpath(__file__))
13-
14-
# ugly trick
15-
ch_path = os.path.join(os.path.dirname(test_root), "BuildLinux/ch")
1639

17-
if not os.path.isfile(ch_path):
18-
print "BuildLinux/ch not found. Did you run ./build.sh already?"
40+
test_root = os.path.dirname(os.path.realpath(__file__))
41+
repo_root = os.path.dirname(test_root)
42+
43+
# arch: x86, x64
44+
arch = 'x86' if args.x86 else ('x64' if args.x64 else None)
45+
if arch == None:
46+
arch = os.environ.get('_BuildArch', 'x86')
47+
48+
# flavor: debug, test, release
49+
flavor = 'debug' if args.debug else ('test' if args.test else None)
50+
if flavor == None:
51+
flavor = {'chk':'debug', 'test':'test', 'fre':'release'}\
52+
[os.environ.get('_BuildType', 'fre')]
53+
54+
# binary: full ch path
55+
binary = args.binary
56+
if binary == None:
57+
if sys.platform == 'win32':
58+
binary = os.path.join(repo_root,
59+
'Build/VcBuild/bin/{}_{}/ch.exe'.format(arch, flavor))
60+
else:
61+
binary = os.path.join(repo_root, "BuildLinux/ch")
62+
if not os.path.isfile(binary):
63+
print '{} not found. Did you run ./build.sh already?'.format(binary)
1964
sys.exit(1)
2065

21-
if len(sys.argv) > 1:
22-
if sys.argv[1] in ['-?', '--help']:
23-
print "ChakraCore *nix Test Script\n"
24-
25-
print "Usage:"
26-
print "test.py <optional test path>\n"
27-
28-
print "-?, --help : Show help\n"
29-
30-
print "Samples:"
31-
print "test only Array:"
32-
print "\t./test.py Array\n"
33-
34-
print "test a single file:"
35-
print "\t./test.py Basics/hello.js\n"
36-
37-
print "test all folders:"
38-
print "\t./test.py"
39-
sys.exit(0)
40-
test_all = None
41-
42-
test_dirs=['']
43-
if test_all:
44-
test_dirs = os.listdir(test_root)
45-
else:
46-
test_dirs[0] = sys.argv[1]
66+
pass_count = 0
67+
fail_count = 0
4768

4869
def show_failed(filename, output, exit_code, expected_output):
4970
print "\nFailed ->", filename
@@ -58,7 +79,7 @@ def show_failed(filename, output, exit_code, expected_output):
5879
ln = min(len(lst_output), len(lst_expected))
5980
for i in range(0, ln):
6081
if lst_output[i] != lst_expected[i]:
61-
print "Output: (at line " + str(i) + ")"
82+
print "Output: (at line " + str(i) + ")"
6283
print "----------------------------"
6384
print lst_output[i]
6485
print "----------------------------"
@@ -69,52 +90,88 @@ def show_failed(filename, output, exit_code, expected_output):
6990
break
7091

7192
print "exit code:", exit_code
72-
print "\nFailed!"
73-
sys.exit(exit_code)
74-
75-
def test_path(folder, is_file):
76-
files=['']
77-
if is_file == False:
78-
print "Testing ->", os.path.basename(folder)
79-
files = os.listdir(folder)
93+
global fail_count
94+
fail_count += 1
95+
96+
def test_path(path):
97+
if os.path.isfile(path):
98+
folder, file = os.path.dirname(path), os.path.basename(path)
8099
else:
81-
files[0] = folder
82-
83-
for js_file in files:
84-
if is_file or os.path.splitext(js_file)[1] == '.js':
85-
js_file = os.path.join(folder, js_file)
86-
js_output = ""
87-
88-
if not os.path.isfile(js_file):
89-
print "Javascript file doesn't exist (" + js_file + ")"
90-
sys.exit(1)
91-
92-
p = SP.Popen([ch_path, js_file], stdout=SP.PIPE, stderr=SP.STDOUT, close_fds=True)
93-
js_output = p.communicate()[0].replace('\r','')
94-
exit_code = p.wait()
95-
96-
if exit_code != 0:
97-
show_failed(js_file, js_output, exit_code, None)
98-
else: #compare outputs
99-
baseline = os.path.splitext(js_file)[0] + '.baseline'
100-
baseline = os.path.join(folder, baseline)
101-
if os.path.isfile(baseline):
102-
expected_output = None
103-
with open(baseline, 'r') as bs_file:
104-
expected_output = bs_file.read().replace('\r', '')
105-
# todo: compare line by line and use/implement wild cards support
106-
# todo: by default we discard line endings (xplat), make this optional
107-
if expected_output.replace('\n', '') != js_output.replace('\n', ''):
108-
show_failed(js_file, js_output, exit_code, expected_output)
109-
110-
if not is_file:
111-
print "\tPassed ->", os.path.basename(js_file)
112-
113-
is_file = len(test_dirs) == 1 and os.path.splitext(test_dirs[0])[1] == '.js'
114-
115-
for folder in test_dirs:
116-
full_path = os.path.join(test_root, folder)
117-
if os.path.isdir(full_path) or is_file:
118-
test_path(full_path, is_file)
119-
120-
print 'Success!'
100+
folder, file = path, None
101+
102+
tests = load_tests(folder, file)
103+
if len(tests) == 0:
104+
return
105+
106+
print "Testing ->", os.path.basename(folder)
107+
for test in tests:
108+
test_one(folder, test)
109+
110+
def test_one(folder, test):
111+
js_file = os.path.join(folder, test['files'])
112+
js_output = ""
113+
114+
cmd = [x for x in [binary,
115+
'-WERExceptionSupport',
116+
'-ExtendedErrorStackForTestHost',
117+
test.get('compile-flags'),
118+
js_file] if x != None]
119+
p = SP.Popen(cmd, stdout=SP.PIPE, stderr=SP.STDOUT)
120+
js_output = p.communicate()[0].replace('\r','')
121+
exit_code = p.wait()
122+
123+
if exit_code != 0:
124+
return show_failed(js_file, js_output, exit_code, None)
125+
else: #compare outputs
126+
baseline = os.path.splitext(js_file)[0] + '.baseline'
127+
if os.path.isfile(baseline):
128+
expected_output = None
129+
with open(baseline, 'r') as bs_file:
130+
expected_output = bs_file.read().replace('\r', '')
131+
# todo: compare line by line and use/implement wild cards support
132+
# todo: by default we discard line endings (xplat), make this optional
133+
if expected_output.replace('\n', '') != js_output.replace('\n', ''):
134+
return show_failed(js_file, js_output, exit_code, expected_output)
135+
136+
print "\tPassed ->", os.path.basename(js_file)
137+
global pass_count
138+
pass_count += 1
139+
140+
def load_tests(folder, file):
141+
try:
142+
xmlpath = os.path.join(folder, 'rlexe.xml')
143+
xml = ET.parse(xmlpath).getroot()
144+
except IOError:
145+
return []
146+
147+
tests = [load_test(x) for x in xml]
148+
if file != None:
149+
tests = [x for x in tests if x['files'] == file]
150+
if len(tests) == 0 and is_jsfile(file):
151+
tests = [{'files':file}]
152+
return tests
153+
154+
def load_test(testXml):
155+
test = dict()
156+
for c in testXml.find('default'):
157+
test[c.tag] = c.text
158+
return test
159+
160+
def is_jsfile(path):
161+
return os.path.splitext(path)[1] == '.js'
162+
163+
def main():
164+
# By default run all tests
165+
if len(args.folders) == 0:
166+
args.folders = [os.path.join(test_root, x)
167+
for x in sorted(os.listdir(test_root))]
168+
169+
for folder in args.folders:
170+
test_path(folder)
171+
172+
print 'Passed:', pass_count, 'Failed:', fail_count
173+
print 'Success!' if fail_count == 0 else 'Failed!'
174+
return 0
175+
176+
if __name__ == '__main__':
177+
sys.exit(main())

0 commit comments

Comments
 (0)