Skip to content

Commit a7e2d4b

Browse files
wesmwesm
committed
Draft testing harness
Change-Id: Ifa749bee89654da257a61a6f2f5255abd4ec5a38
1 parent 86f56a6 commit a7e2d4b

File tree

5 files changed

+228
-10
lines changed

5 files changed

+228
-10
lines changed

‎.gitignore‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
# Compiled source
19+
*.a
20+
*.dll
21+
*.o
22+
*.py[ocd]
23+
*.so
24+
*.dylib
25+
.build_cache_dir
26+
MANIFEST

‎integration/data/simple.json‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
{
2+
"schema": {
3+
"fields": [
4+
{
5+
"name": "foo",
6+
"type": {"name": "int", "isSigned": true, "bitWidth": 32},
7+
"nullable": true, "children": [],
8+
"typeLayout": [
9+
{"type": "VALIDITY", "typeBitWidth": 1},
10+
{"type": "DATA", "typeBitWidth": 32}
11+
]
12+
},
13+
{
14+
"name": "bar",
15+
"type": {"name": "floatingpoint", "precision": "DOUBLE"},
16+
"nullable": true, "children": [],
17+
"typeLayout": [
18+
{"type": "VALIDITY", "typeBitWidth": 1},
19+
{"type": "DATA", "typeBitWidth": 64}
20+
]
21+
}
22+
]
23+
},
24+
"batches": [
25+
{
26+
"count": 5,
27+
"columns": [
28+
{
29+
"name": "foo",
30+
"count": 5,
31+
"DATA": [1, 2, 3, 4, 5],
32+
"VALIDITY": [1, 0, 1, 1, 1]
33+
},
34+
{
35+
"name": "bar",
36+
"count": 5,
37+
"DATA": [1.0, 2.0, 3.0, 4.0, 5.0],
38+
"VALIDITY": [1, 0, 0, 1, 1]
39+
}
40+
]
41+
}
42+
]
43+
}

‎integration/integration_test.py‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
import glob
19+
import itertools
20+
import os
21+
import six
22+
import subprocess
23+
import tempfile
24+
import uuid
25+
26+
27+
ARROW_HOME = os.path.abspath(__file__).rsplit("/", 2)[0]
28+
29+
30+
def guid():
31+
return uuid.uuid4().hex
32+
33+
34+
def run_cmd(cmd):
35+
if isinstance(cmd, six.string_types):
36+
cmd = cmd.split(' ')
37+
38+
try:
39+
output = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
40+
except subprocess.CalledProcessError as e:
41+
# this avoids hiding the stdout / stderr of failed processes
42+
print('Command failed: %s' % ' '.join(cmd))
43+
print('With output:')
44+
print('--------------')
45+
print(e.output)
46+
print('--------------')
47+
raise e
48+
49+
if isinstance(output, six.binary_type):
50+
output = output.decode('utf-8')
51+
return output
52+
53+
54+
class IntegrationRunner(object):
55+
56+
def __init__(self, json_files, testers):
57+
self.json_files = json_files
58+
self.testers = testers
59+
self.temp_dir = tempfile.mkdtemp()
60+
61+
def run(self):
62+
for producer, consumer in itertools.product(self.testers,
63+
self.testers):
64+
if producer is consumer:
65+
continue
66+
67+
print('-- {0} producing, {1} consuming'.format(producer.name,
68+
consumer.name))
69+
70+
for json_path in self.json_files:
71+
print('Testing with {0}'.format(json_path))
72+
73+
arrow_path = os.path.join(self.temp_dir, guid())
74+
75+
producer.json_to_arrow(json_path, arrow_path)
76+
consumer.validate(json_path, arrow_path)
77+
78+
79+
class Tester(object):
80+
81+
def json_to_arrow(self, json_path, arrow_path):
82+
raise NotImplementedError
83+
84+
def validate(self, json_path, arrow_path):
85+
raise NotImplementedError
86+
87+
88+
class JavaTester(Tester):
89+
90+
ARROW_TOOLS_JAR = os.path.join(ARROW_HOME,
91+
'java/tools/target/arrow-tools-0.1.1-'
92+
'SNAPSHOT-jar-with-dependencies.jar')
93+
94+
name = 'Java'
95+
96+
def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
97+
cmd = ['java', '-cp', self.ARROW_TOOLS_JAR,
98+
'org.apache.arrow.tools.Integration']
99+
100+
if arrow_path is not None:
101+
cmd.extend(['-a', arrow_path])
102+
103+
if json_path is not None:
104+
cmd.extend(['-j', json_path])
105+
106+
cmd.extend(['-c', command])
107+
return run_cmd(cmd)
108+
109+
def validate(self, json_path, arrow_path):
110+
return self._run(arrow_path, json_path, 'VALIDATE')
111+
112+
def json_to_arrow(self, json_path, arrow_path):
113+
return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
114+
115+
116+
class CPPTester(Tester):
117+
118+
CPP_INTEGRATION_EXE = os.environ.get(
119+
'ARROW_CPP_TESTER',
120+
os.path.join(ARROW_HOME,
121+
'cpp/test-build/debug/json-integration-test'))
122+
123+
name = 'C++'
124+
125+
def _run(self, arrow_path=None, json_path=None, command='VALIDATE'):
126+
cmd = [self.CPP_INTEGRATION_EXE, '--integration']
127+
128+
if arrow_path is not None:
129+
cmd.append('--arrow=' + arrow_path)
130+
131+
if json_path is not None:
132+
cmd.append('--json=' + json_path)
133+
134+
cmd.append('--mode=' + command)
135+
return run_cmd(cmd)
136+
137+
def validate(self, json_path, arrow_path):
138+
return self._run(arrow_path, json_path, 'VALIDATE')
139+
140+
def json_to_arrow(self, json_path, arrow_path):
141+
return self._run(arrow_path, json_path, 'JSON_TO_ARROW')
142+
143+
144+
def get_json_files():
145+
glob_pattern = os.path.join(ARROW_HOME, 'integration', 'data', '*.json')
146+
return glob.glob(glob_pattern)
147+
148+
149+
def run_all_tests():
150+
testers = [JavaTester(), CPPTester()]
151+
json_files = get_json_files()
152+
153+
runner = IntegrationRunner(json_files, testers)
154+
runner.run()
155+
156+
157+
if __name__ == '__main__':
158+
run_all_tests()

‎java/tools/src/main/java/org/apache/arrow/tools/Integration.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -220,6 +220,7 @@ private Command toCommand(String commandName) {
220220

221221
private static void fatalError(String message, Throwable e) {
222222
System.err.println(message);
223+
System.err.println(e.getMessage());
223224
LOGGER.error(message, e);
224225
System.exit(1);
225226
}

‎python/.gitignore‎

Lines changed: 0 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,16 +12,6 @@ Testing/
1212
# Editor temporary/working/backup files
1313
*flymake*
1414

15-
# Compiled source
16-
*.a
17-
*.dll
18-
*.o
19-
*.py[ocd]
20-
*.so
21-
*.dylib
22-
.build_cache_dir
23-
MANIFEST
24-
2515
# Generated sources
2616
*.c
2717
*.cpp

0 commit comments

Comments
 (0)