-
Notifications
You must be signed in to change notification settings - Fork 461
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add performance benchmarking scripts
And run them by Travis-CI
- Loading branch information
Showing
5 changed files
with
307 additions
and
3 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,119 @@ | ||
| #!/usr/bin/env python | ||
| # -*- coding: utf-8 -*- | ||
| # | ||
| # Copyright (c) 2017, IntoPIX SA | ||
| # Contact: support@intopix.com | ||
| # Author: Even Rouault | ||
| # | ||
| # Redistribution and use in source and binary forms, with or without | ||
| # modification, are permitted provided that the following conditions | ||
| # are met: | ||
| # 1. Redistributions of source code must retain the above copyright | ||
| # notice, this list of conditions and the following disclaimer. | ||
| # 2. Redistributions in binary form must reproduce the above copyright | ||
| # notice, this list of conditions and the following disclaimer in the | ||
| # documentation and/or other materials provided with the distribution. | ||
| # | ||
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS `AS IS' | ||
| # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
| # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| # POSSIBILITY OF SUCH DAMAGE. | ||
| # | ||
|
|
||
| import sys | ||
|
|
||
|
|
||
| def Usage(): | ||
| print('Usage: compare_perfs.py [-noise_threshold val_in_pct]') | ||
| print(' [-warning_threshold val_in_pct]') | ||
| print(' [-error_threshold val_in_pct]') | ||
| print(' [-global_error_threshold val_in_pct]') | ||
| print(' ref.csv new.csv') | ||
| sys.exit(1) | ||
|
|
||
| ref_filename = None | ||
| new_filename = None | ||
| noise_threshold = 2 | ||
| warning_threshold = 4 | ||
| error_threshold = 6 | ||
| global_error_threshold = 1 | ||
| i = 1 | ||
| while i < len(sys.argv): | ||
| if sys.argv[i] == '-noise_threshold' and i + 1 < len(sys.argv): | ||
| i += 1 | ||
| noise_threshold = int(sys.argv[i]) | ||
| elif sys.argv[i] == '-warning_threshold' and i + 1 < len(sys.argv): | ||
| i += 1 | ||
| warning_threshold = int(sys.argv[i]) | ||
| elif sys.argv[i] == '-error_threshold' and i + 1 < len(sys.argv): | ||
| i += 1 | ||
| error_threshold = int(sys.argv[i]) | ||
| elif sys.argv[i] == '-global_error_threshold' and i + 1 < len(sys.argv): | ||
| i += 1 | ||
| global_error_threshold = int(sys.argv[i]) | ||
| elif sys.argv[i][0] == '-': | ||
| Usage() | ||
| elif ref_filename is None: | ||
| ref_filename = sys.argv[i] | ||
| elif new_filename is None: | ||
| new_filename = sys.argv[i] | ||
| else: | ||
| Usage() | ||
| i += 1 | ||
| if ref_filename is None or new_filename is None: | ||
| Usage() | ||
|
|
||
| assert noise_threshold < warning_threshold | ||
| assert warning_threshold < error_threshold | ||
| assert global_error_threshold <= error_threshold | ||
|
|
||
| ref_lines = open(ref_filename, 'rt').readlines()[1:] | ||
| new_lines = open(new_filename, 'rt').readlines()[1:] | ||
| if len(ref_lines) != len(new_lines): | ||
| raise Exception('files are not comparable') | ||
|
|
||
| ret_code = 0 | ||
| for i in range(len(ref_lines)): | ||
| line = ref_lines[i].replace('\n', '') | ||
| filename_ref, num_iterations_ref, num_threads_ref, command_ref, \ | ||
| _, time_ms_ref = line.split(',') | ||
| line = new_lines[i].replace('\n', '') | ||
| filename_new, num_iterations_new, num_threads_new, command_new, \ | ||
| _, time_ms_new = line.split(',') | ||
| assert filename_ref == filename_new | ||
| assert num_iterations_ref == num_iterations_new | ||
| assert num_threads_ref == num_threads_new | ||
| assert command_ref == command_new | ||
| time_ms_ref = int(time_ms_ref) | ||
| time_ms_new = int(time_ms_new) | ||
| if filename_ref == 'TOTAL': | ||
| display = 'TOTAL' | ||
| else: | ||
| display = '%s, %s iterations, %s threads, %s' % \ | ||
| (filename_ref, num_iterations_ref, num_threads_ref, command_ref) | ||
| display += ': ref_time %d ms, new_time %d ms' % (time_ms_ref, time_ms_new) | ||
| var_pct = 100.0 * (time_ms_new - time_ms_ref) / time_ms_ref | ||
| if abs(var_pct) <= noise_threshold: | ||
| display += ', (stable) %0.1f %%' % var_pct | ||
| elif var_pct < 0: | ||
| display += ', (improvement) %0.1f %%' % var_pct | ||
| else: | ||
| display += ', (regression) %0.1f %%' % var_pct | ||
| if filename_ref == 'TOTAL' and var_pct > global_error_threshold: | ||
| display += ', ERROR_THRESHOLD' | ||
| ret_code = 1 | ||
| elif var_pct > error_threshold: | ||
| display += ', ERROR_THRESHOLD' | ||
| ret_code = 1 | ||
| elif var_pct > warning_threshold: | ||
| display += ', WARNING_THRESHOLD' | ||
| print(display) | ||
|
|
||
| sys.exit(ret_code) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| #!/usr/bin/env python | ||
| # -*- coding: utf-8 -*- | ||
| # | ||
| # Copyright (c) 2017, IntoPIX SA | ||
| # Contact: support@intopix.com | ||
| # Author: Even Rouault | ||
| # | ||
| # Redistribution and use in source and binary forms, with or without | ||
| # modification, are permitted provided that the following conditions | ||
| # are met: | ||
| # 1. Redistributions of source code must retain the above copyright | ||
| # notice, this list of conditions and the following disclaimer. | ||
| # 2. Redistributions in binary form must reproduce the above copyright | ||
| # notice, this list of conditions and the following disclaimer in the | ||
| # documentation and/or other materials provided with the distribution. | ||
| # | ||
| # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS `AS IS' | ||
| # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE | ||
| # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE | ||
| # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE | ||
| # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR | ||
| # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF | ||
| # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS | ||
| # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN | ||
| # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) | ||
| # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE | ||
| # POSSIBILITY OF SUCH DAMAGE. | ||
| # | ||
|
|
||
| import os | ||
| import subprocess | ||
| import sys | ||
| import time | ||
|
|
||
|
|
||
| def Usage(): | ||
| print('Usage: perf_test.py [-kakadu] [-i filelist.csv] [-o out.csv] [-q]') | ||
| sys.exit(1) | ||
|
|
||
| opj_decompress_path = 'opj_decompress' | ||
| opj_compress_path = 'opj_compress' | ||
| kdu_expand_path = 'kdu_expand' | ||
| kdu_compress_path = 'kdu_compress' | ||
|
|
||
| in_filename = 'perf_test_filelist.csv' | ||
| out_filename = None | ||
| i = 1 | ||
| quiet = False | ||
| kakadu = False | ||
| while i < len(sys.argv): | ||
| if sys.argv[i] == '-o' and i + 1 < len(sys.argv): | ||
| i += 1 | ||
| out_filename = sys.argv[i] | ||
| elif sys.argv[i] == '-q': | ||
| quiet = True | ||
| elif sys.argv[i] == '-kakadu': | ||
| kakadu = True | ||
| else: | ||
| Usage() | ||
| i += 1 | ||
|
|
||
| i = 0 | ||
| while i < 10 * 1024 * 1024: | ||
| i += 1 | ||
|
|
||
| out_file = None | ||
| if out_filename is not None: | ||
| out_file = open(out_filename, 'wt') | ||
| out_file.write('filename,iterations,threads,command,comment,time_ms\n') | ||
|
|
||
| total_time = 0 | ||
| for line in open(in_filename, 'rt').readlines()[1:]: | ||
| line = line.replace('\n', '') | ||
| filename, num_iterations, num_threads, command, comment = line.split(',') | ||
| num_threads = int(num_threads) | ||
| num_iterations = int(num_iterations) | ||
| start = time.time() | ||
| for i in range(num_iterations): | ||
| env = None | ||
| if kakadu: | ||
| if command == 'DECOMPRESS': | ||
| args = [kdu_expand_path, | ||
| '-i', filename, | ||
| '-num_threads', str(num_threads), | ||
| '-o', 'out_perf_test.pgm'] | ||
| elif command == 'COMPRESS': | ||
| args = [kdu_compress_path, | ||
| '-i', filename, | ||
| '-num_threads', str(num_threads), | ||
| 'Creversible=yes', | ||
| '-o', 'out_perf_test.jp2'] | ||
| else: | ||
| assert False, command | ||
| else: | ||
| env = os.environ | ||
| if num_threads > 1: | ||
| env['OPJ_NUM_THREADS'] = str(num_threads) | ||
| else: | ||
| env['OPJ_NUM_THREADS'] = '0' | ||
| if command == 'DECOMPRESS': | ||
| args = [opj_decompress_path, | ||
| '-i', filename, '-o', 'out_perf_test.pgm'] | ||
| elif command == 'COMPRESS': | ||
| args = [opj_compress_path, | ||
| '-i', filename, '-o', 'out_perf_test.jp2'] | ||
| else: | ||
| assert False, command | ||
| p = subprocess.Popen(args, | ||
| stdout=subprocess.PIPE, | ||
| stderr=subprocess.PIPE, | ||
| close_fds=True, | ||
| env=env) | ||
| p.wait() | ||
| stop = time.time() | ||
| if os.path.exists('out_perf_test.pgm'): | ||
| os.unlink('out_perf_test.pgm') | ||
| if os.path.exists('out_perf_test.jp2'): | ||
| os.unlink('out_perf_test.jp2') | ||
| spent_time = stop - start | ||
| total_time += spent_time | ||
| if not quiet: | ||
| if len(comment) != 0: | ||
| print('%s (%s), %d iterations, %d threads, %s: %.02f s' % | ||
| (filename, comment, num_iterations, num_threads, | ||
| command, spent_time)) | ||
| else: | ||
| print('%s, %d iterations, %d threads, %s: %.02f s' % | ||
| (filename, num_iterations, num_threads, command, spent_time)) | ||
| if out_file is not None: | ||
| out_file.write('%s,%d,%d,%s,%s,%d\n' % | ||
| (filename, num_iterations, num_threads, command, | ||
| comment, spent_time * 1000)) | ||
|
|
||
| if not quiet: | ||
| print('Total time: %.02f s' % total_time) | ||
| if out_file is not None: | ||
| out_file.write('%s,,,,,%d\n' % ('TOTAL', total_time * 1000)) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,7 @@ | ||
| filename,iterations,threads,command,comment | ||
| ../../data/input/nonregression/kodak_2layers_lrcp.j2c,3,1,DECOMPRESS, | ||
| ../../data/input/nonregression/kodak_2layers_lrcp.j2c,5,2,DECOMPRESS, | ||
| ../../data/input/nonregression/kodak_2layers_lrcp.j2c,10,4,DECOMPRESS, | ||
| ../../data/input/conformance/p0_07.j2k,3,1,DECOMPRESS,128x128 tiles RLCP 3decomp 8layers no MCT reversible | ||
| ../../data/input/conformance/p0_04.j2k,10,1,DECOMPRESS,precincts 128x128 irreversible | ||
| ../../data/input/nonregression/X_4_2K_24_185_CBR_WB_000.tif,3,1,COMPRESS, |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters