Skip to content

Commit

Permalink
Apply UFMT to all files in benchmarks/ (pytorch#105928)
Browse files Browse the repository at this point in the history 
Signed-off-by: Edward Z. Yang <ezyang@meta.com>

Pull Request resolved: pytorch#105928
Approved by: https://github.com/albanD
  • Loading branch information
@ezyang @pytorchmergebot
ezyang authored and pytorchmergebot committed Jul 26, 2023
1 parent a361fce commit dd3a77b
Show file tree
Hide file tree
Showing 181 changed files with 5,613 additions and 3,897 deletions.
203 changes: 0 additions & 203 deletions .lintrunner.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -949,209 +949,6 @@ exclude_patterns = [
'aten/src/ATen/native/quantized/cpu/qnnpack/generate-wrapper.py',
'aten/src/ATen/native/transformers/cuda/mem_eff_attention/kernels/generate_kernels.py',
'aten/src/ATen/nnapi/codegen.py',
'benchmarks/compare-fastrnn-results.py',
'benchmarks/cpp/tensorexpr/bench_ops.py',
'benchmarks/distributed/ddp/benchmark.py',
'benchmarks/distributed/ddp/diff.py',
'benchmarks/distributed/pipeline/benchmark_dataset.py',
'benchmarks/distributed/pipeline/pipe.py',
'benchmarks/distributed/rpc/parameter_server/data/DummyData.py',
'benchmarks/distributed/rpc/parameter_server/data/__init__.py',
'benchmarks/distributed/rpc/parameter_server/launcher.py',
'benchmarks/distributed/rpc/parameter_server/metrics/CPUMetric.py',
'benchmarks/distributed/rpc/parameter_server/metrics/CUDAMetric.py',
'benchmarks/distributed/rpc/parameter_server/metrics/MetricBase.py',
'benchmarks/distributed/rpc/parameter_server/metrics/MetricsLogger.py',
'benchmarks/distributed/rpc/parameter_server/metrics/ProcessedMetricsPrinter.py',
'benchmarks/distributed/rpc/parameter_server/models/DummyModel.py',
'benchmarks/distributed/rpc/parameter_server/models/__init__.py',
'benchmarks/distributed/rpc/parameter_server/server/__init__.py',
'benchmarks/distributed/rpc/parameter_server/server/server.py',
'benchmarks/distributed/rpc/parameter_server/trainer/__init__.py',
'benchmarks/distributed/rpc/parameter_server/trainer/criterions.py',
'benchmarks/distributed/rpc/parameter_server/trainer/ddp_models.py',
'benchmarks/distributed/rpc/parameter_server/trainer/hook_states.py',
'benchmarks/distributed/rpc/parameter_server/trainer/hooks.py',
'benchmarks/distributed/rpc/parameter_server/trainer/iteration_steps.py',
'benchmarks/distributed/rpc/parameter_server/trainer/preprocess_data.py',
'benchmarks/distributed/rpc/parameter_server/trainer/trainer.py',
'benchmarks/distributed/rpc/parameter_server/utils.py',
'benchmarks/distributed/rpc/rl/agent.py',
'benchmarks/distributed/rpc/rl/coordinator.py',
'benchmarks/distributed/rpc/rl/launcher.py',
'benchmarks/distributed/rpc/rl/observer.py',
'benchmarks/fastrnns/__init__.py',
'benchmarks/fastrnns/bench.py',
'benchmarks/fastrnns/cells.py',
'benchmarks/fastrnns/conftest.py',
'benchmarks/fastrnns/custom_lstms.py',
'benchmarks/fastrnns/factory.py',
'benchmarks/fastrnns/fuser.py',
'benchmarks/fastrnns/profile.py',
'benchmarks/fastrnns/runner.py',
'benchmarks/fastrnns/scratch.py',
'benchmarks/fastrnns/test.py',
'benchmarks/fastrnns/test_bench.py',
'benchmarks/framework_overhead_benchmark/C2Module.py',
'benchmarks/framework_overhead_benchmark/SimpleAddModule.py',
'benchmarks/framework_overhead_benchmark/framework_overhead_benchmark.py',
'benchmarks/framework_overhead_benchmark/pt_wrapper_module.py',
'benchmarks/framework_overhead_benchmark/utils.py',
'benchmarks/functional_autograd_benchmark/audio_text_models.py',
'benchmarks/functional_autograd_benchmark/compare.py',
'benchmarks/functional_autograd_benchmark/functional_autograd_benchmark.py',
'benchmarks/functional_autograd_benchmark/ppl_models.py',
'benchmarks/functional_autograd_benchmark/torchaudio_models.py',
'benchmarks/functional_autograd_benchmark/torchvision_models.py',
'benchmarks/functional_autograd_benchmark/utils.py',
'benchmarks/functional_autograd_benchmark/vision_models.py',
'benchmarks/fuser/plot_speedups.py',
'benchmarks/fuser/run_benchmarks.py',
'benchmarks/instruction_counts/applications/__init__.py',
'benchmarks/instruction_counts/applications/ci.py',
'benchmarks/instruction_counts/core/__init__.py',
'benchmarks/instruction_counts/core/api.py',
'benchmarks/instruction_counts/core/expand.py',
'benchmarks/instruction_counts/core/types.py',
'benchmarks/instruction_counts/core/utils.py',
'benchmarks/instruction_counts/definitions/__init__.py',
'benchmarks/instruction_counts/definitions/setup.py',
'benchmarks/instruction_counts/definitions/standard.py',
'benchmarks/instruction_counts/execution/__init__.py',
'benchmarks/instruction_counts/execution/runner.py',
'benchmarks/instruction_counts/execution/work.py',
'benchmarks/instruction_counts/main.py',
'benchmarks/instruction_counts/worker/__init__.py',
'benchmarks/instruction_counts/worker/main.py',
'benchmarks/nested/nested_bmm_bench.py',
'benchmarks/operator_benchmark/__init__.py',
'benchmarks/operator_benchmark/benchmark_all_other_test.py',
'benchmarks/operator_benchmark/benchmark_all_quantized_test.py',
'benchmarks/operator_benchmark/benchmark_all_test.py',
'benchmarks/operator_benchmark/benchmark_caffe2.py',
'benchmarks/operator_benchmark/benchmark_core.py',
'benchmarks/operator_benchmark/benchmark_pytorch.py',
'benchmarks/operator_benchmark/benchmark_runner.py',
'benchmarks/operator_benchmark/benchmark_test_generator.py',
'benchmarks/operator_benchmark/benchmark_utils.py',
'benchmarks/operator_benchmark/c2/__init__.py',
'benchmarks/operator_benchmark/c2/add_test.py',
'benchmarks/operator_benchmark/c2/batch_box_cox_test.py',
'benchmarks/operator_benchmark/c2/batch_gather_test.py',
'benchmarks/operator_benchmark/c2/clip_ranges_test.py',
'benchmarks/operator_benchmark/c2/concat_test.py',
'benchmarks/operator_benchmark/c2/matmul_test.py',
'benchmarks/operator_benchmark/c2/quantile_op_test.py',
'benchmarks/operator_benchmark/c2/replace_nan_test.py',
'benchmarks/operator_benchmark/common/__init__.py',
'benchmarks/operator_benchmark/common/repeat_benchmark.py',
'benchmarks/operator_benchmark/common/tests/add_ops_list_test.py',
'benchmarks/operator_benchmark/common/tests/c2_cpu_gpu_forward_backward_test.py',
'benchmarks/operator_benchmark/common/tests/jit_forward_test.py',
'benchmarks/operator_benchmark/common/tests/pt_backward_test.py',
'benchmarks/operator_benchmark/common/tests/pt_configs_list_test.py',
'benchmarks/operator_benchmark/common/tests/pt_cpu_gpu_forward_backward_test.py',
'benchmarks/operator_benchmark/common/tests/random_sample_test.py',
'benchmarks/operator_benchmark/operator_benchmark.py',
'benchmarks/operator_benchmark/pt/__init__.py',
'benchmarks/operator_benchmark/pt/add_test.py',
'benchmarks/operator_benchmark/pt/ao_sparsifier_test.py',
'benchmarks/operator_benchmark/pt/as_strided_test.py',
'benchmarks/operator_benchmark/pt/batchnorm_test.py',
'benchmarks/operator_benchmark/pt/binary_test.py',
'benchmarks/operator_benchmark/pt/bmm_test.py',
'benchmarks/operator_benchmark/pt/cat_test.py',
'benchmarks/operator_benchmark/pt/channel_shuffle_test.py',
'benchmarks/operator_benchmark/pt/chunk_test.py',
'benchmarks/operator_benchmark/pt/clip_ranges_test.py',
'benchmarks/operator_benchmark/pt/configs.py',
'benchmarks/operator_benchmark/pt/conv_test.py',
'benchmarks/operator_benchmark/pt/diag_test.py',
'benchmarks/operator_benchmark/pt/embeddingbag_test.py',
'benchmarks/operator_benchmark/pt/fill_test.py',
'benchmarks/operator_benchmark/pt/gather_test.py',
'benchmarks/operator_benchmark/pt/gelu_test.py',
'benchmarks/operator_benchmark/pt/groupnorm_test.py',
'benchmarks/operator_benchmark/pt/hardsigmoid_test.py',
'benchmarks/operator_benchmark/pt/hardswish_test.py',
'benchmarks/operator_benchmark/pt/index_select_test.py',
'benchmarks/operator_benchmark/pt/instancenorm_test.py',
'benchmarks/operator_benchmark/pt/interpolate_test.py',
'benchmarks/operator_benchmark/pt/layernorm_test.py',
'benchmarks/operator_benchmark/pt/linear_prepack_fp16_test.py',
'benchmarks/operator_benchmark/pt/linear_test.py',
'benchmarks/operator_benchmark/pt/linear_unpack_fp16_test.py',
'benchmarks/operator_benchmark/pt/matmul_test.py',
'benchmarks/operator_benchmark/pt/matrix_mult_test.py',
'benchmarks/operator_benchmark/pt/nan_to_num_test.py',
'benchmarks/operator_benchmark/pt/pool_test.py',
'benchmarks/operator_benchmark/pt/qactivation_test.py',
'benchmarks/operator_benchmark/pt/qarithmetic_test.py',
'benchmarks/operator_benchmark/pt/qatembedding_ops_test.py',
'benchmarks/operator_benchmark/pt/qbatchnorm_test.py',
'benchmarks/operator_benchmark/pt/qcat_test.py',
'benchmarks/operator_benchmark/pt/qcomparators_test.py',
'benchmarks/operator_benchmark/pt/qconv_test.py',
'benchmarks/operator_benchmark/pt/qembedding_bag_lookups_test.py',
'benchmarks/operator_benchmark/pt/qembedding_pack_test.py',
'benchmarks/operator_benchmark/pt/qembeddingbag_test.py',
'benchmarks/operator_benchmark/pt/qgroupnorm_test.py',
'benchmarks/operator_benchmark/pt/qinstancenorm_test.py',
'benchmarks/operator_benchmark/pt/qinterpolate_test.py',
'benchmarks/operator_benchmark/pt/qlayernorm_test.py',
'benchmarks/operator_benchmark/pt/qlinear_test.py',
'benchmarks/operator_benchmark/pt/qobserver_test.py',
'benchmarks/operator_benchmark/pt/qpool_test.py',
'benchmarks/operator_benchmark/pt/qrnn_test.py',
'benchmarks/operator_benchmark/pt/qtensor_method_test.py',
'benchmarks/operator_benchmark/pt/quantization_test.py',
'benchmarks/operator_benchmark/pt/qunary_test.py',
'benchmarks/operator_benchmark/pt/remainder_test.py',
'benchmarks/operator_benchmark/pt/softmax_test.py',
'benchmarks/operator_benchmark/pt/split_test.py',
'benchmarks/operator_benchmark/pt/stack_test.py',
'benchmarks/operator_benchmark/pt/sum_test.py',
'benchmarks/operator_benchmark/pt/tensor_to_test.py',
'benchmarks/operator_benchmark/pt/unary_test.py',
'benchmarks/operator_benchmark/pt_extension/cpp_extension_test.py',
'benchmarks/operator_benchmark/pt_extension/setup.py',
'benchmarks/overrides_benchmark/bench.py',
'benchmarks/overrides_benchmark/common.py',
'benchmarks/overrides_benchmark/pyspybench.py',
'benchmarks/profiler_benchmark/profiler_bench.py',
'benchmarks/profiler_benchmark/resnet_memory_profiler.py',
'benchmarks/record_function_benchmark/record_function_bench.py',
'benchmarks/serialization/nested_annotation_str.py',
'benchmarks/serialization/simple_measurement.py',
'benchmarks/sparse/__init__.py',
'benchmarks/sparse/benchmark_semi_structured_sparsity.py',
'benchmarks/sparse/dlmc/__init__.py',
'benchmarks/sparse/dlmc/matmul_bench.py',
'benchmarks/sparse/dlmc/utils.py',
'benchmarks/sparse/spmm.py',
'benchmarks/sparse/spmv.py',
'benchmarks/sparse/utils.py',
'benchmarks/tensorexpr/__main__.py',
'benchmarks/tensorexpr/attention.py',
'benchmarks/tensorexpr/benchmark.py',
'benchmarks/tensorexpr/broadcast.py',
'benchmarks/tensorexpr/concat.py',
'benchmarks/tensorexpr/conv.py',
'benchmarks/tensorexpr/elementwise.py',
'benchmarks/tensorexpr/matmul.py',
'benchmarks/tensorexpr/microbenchmarks.py',
'benchmarks/tensorexpr/normalization.py',
'benchmarks/tensorexpr/pooling.py',
'benchmarks/tensorexpr/pt_engine.py',
'benchmarks/tensorexpr/reduction.py',
'benchmarks/tensorexpr/rnn_eltwise.py',
'benchmarks/tensorexpr/softmax.py',
'benchmarks/tensorexpr/swish.py',
'benchmarks/tensorexpr/tensor_engine.py',
'benchmarks/transformer/better_transformer_vs_mha_functional.py',
'benchmarks/transformer/sdp.py',
'benchmarks/transformer/sdp_backwards.py',
'benchmarks/upload_scribe.py',
'binaries/bench_gen/bench_gen.py',
'docs/caffe2/process.py',
'docs/cpp/source/conf.py',
Expand Down
48 changes: 32 additions & 16 deletions benchmarks/compare-fastrnn-results.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,13 @@

Result = namedtuple("Result", ["name", "base_time", "diff_time"])


def construct_name(fwd_bwd, test_name):
bwd = 'backward' in fwd_bwd
suite_name = fwd_bwd.replace('-backward', '')
bwd = "backward" in fwd_bwd
suite_name = fwd_bwd.replace("-backward", "")
return f"{suite_name}[{test_name}]:{'bwd' if bwd else 'fwd'}"


def get_times(json_data):
r = {}
for fwd_bwd in json_data:
Expand All @@ -17,10 +19,13 @@ def get_times(json_data):
r[name] = json_data[fwd_bwd][test_name]
return r


parser = argparse.ArgumentParser("compare two pytest jsons")
parser.add_argument('base', help="base json file")
parser.add_argument('diff', help='diff json file')
parser.add_argument('--format', default='md', type=str, help='output format (csv, md, json, table)')
parser.add_argument("base", help="base json file")
parser.add_argument("diff", help="diff json file")
parser.add_argument(
"--format", default="md", type=str, help="output format (csv, md, json, table)"
)
args = parser.parse_args()

with open(args.base) as base:
Expand All @@ -34,22 +39,33 @@ def get_times(json_data):
for name in sorted(all_keys)
]

header_fmt = {'table' : '{:48s} {:>13s} {:>15s} {:>10s}',
'md' : '| {:48s} | {:>13s} | {:>15s} | {:>10s} |',
'csv' : '{:s}, {:s}, {:s}, {:s}'}
data_fmt = {'table' : '{:48s} {:13.6f} {:15.6f} {:9.1f}%',
'md' : '| {:48s} | {:13.6f} | {:15.6f} | {:9.1f}% |',
'csv' : '{:s}, {:.6f}, {:.6f}, {:.2f}%'}
header_fmt = {
"table": "{:48s} {:>13s} {:>15s} {:>10s}",
"md": "| {:48s} | {:>13s} | {:>15s} | {:>10s} |",
"csv": "{:s}, {:s}, {:s}, {:s}",
}
data_fmt = {
"table": "{:48s} {:13.6f} {:15.6f} {:9.1f}%",
"md": "| {:48s} | {:13.6f} | {:15.6f} | {:9.1f}% |",
"csv": "{:s}, {:.6f}, {:.6f}, {:.2f}%",
}

if args.format in ['table', 'md', 'csv']:
if args.format in ["table", "md", "csv"]:
header_fmt_str = header_fmt[args.format]
data_fmt_str = data_fmt[args.format]
print(header_fmt_str.format("name", "base time (s)", "diff time (s)", "% change"))
if args.format == 'md':
if args.format == "md":
print(header_fmt_str.format(":---", "---:", "---:", "---:"))
for r in results:
print(data_fmt_str.format(r.name, r.base_time, r.diff_time, (r.diff_time / r.base_time - 1.0) * 100.0))
elif args.format == 'json':
print(
data_fmt_str.format(
r.name,
r.base_time,
r.diff_time,
(r.diff_time / r.base_time - 1.0) * 100.0,
)
)
elif args.format == "json":
print(json.dumps(results))
else:
raise ValueError('Unknown output format: ' + args.format)
raise ValueError("Unknown output format: " + args.format)
14 changes: 11 additions & 3 deletions benchmarks/cpp/tensorexpr/bench_ops.py
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import timeit

import torch
import torch.nn.functional as F

Expand Down Expand Up @@ -67,6 +68,7 @@ def hardswish(x):
tjit = timeit.timeit(stmt="traced(x)", globals=globals(), number=bench_iters)
print(f"{op.__name__:20s} {teager:10.3f} {tjit:10.3f} {teager/tjit:10.2f}")


def test_batch_norm():
op = F.batch_norm
print(f"{'op':20s} {'shape':20s} {'eager':>10s} {'nnc':>10s} {'speedup':>10s}")
Expand All @@ -80,7 +82,8 @@ def test_batch_norm():
[5, 256, 14, 14],
[5, 128, 28, 28],
[5, 64, 56, 56],
[5, 512, 7, 7]]
[5, 512, 7, 7],
]
for n, c, h, w in batch_norm_shapes:
x = torch.rand((n, c, h, w))
y = torch.rand(c)
Expand All @@ -99,7 +102,12 @@ def test_batch_norm():
# Benchmark.
bench_iters = 100
teager = timeit.timeit(stmt="op(x, y, z)", globals=locals(), number=bench_iters)
tjit = timeit.timeit(stmt="traced(x, y, z)", globals=locals(), number=bench_iters)
print(f"{op.__name__:20s} ({n:>3d}, {c:>3d}, {h:>3d}, {w:>3d}) {teager:10.3f} {tjit:10.3f} {teager/tjit:10.2f}")
tjit = timeit.timeit(
stmt="traced(x, y, z)", globals=locals(), number=bench_iters
)
print(
f"{op.__name__:20s} ({n:>3d}, {c:>3d}, {h:>3d}, {w:>3d}) {teager:10.3f} {tjit:10.3f} {teager/tjit:10.2f}"
)


test_batch_norm()
Loading

0 comments on commit dd3a77b

Please sign in to comment.