ydb-platform · aakulaga-ydb · May 16, 2024 · May 15, 2024 · May 15, 2024 · May 15, 2024
diff --git a/ydb/library/benchmarks/runner/result_compare/compare.py b/ydb/library/benchmarks/runner/result_compare/compare.py
@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+import sys
+import signal
+import traceback
+import html
+import math
+from pathlib import Path
+
+import cyson as yson
+
+
+def main():
+
+    if len(sys.argv) < 2:
+        print('Usage: {} resultdir... >report.htm'.format(sys.argv[0]), file=sys.stderr)
+        sys.exit(1)
+
+    rdirs = sys.argv[1:]
+    data = []
+
+    print('''
+<html><head><style>.signal { color: blue; } .errcode { color: red; } .ok { color: green; } .mismatch { color: yellow; } .tabnum { text-align: right; } </style></head>
+''')
+    print('<table border="1">')
+    print('<tr><th>' + ''.join(map(lambda x: '<th colspan="5">' + html.escape(rdirs[x]), range(len(rdirs)))))
+    print('<tr><th>Testcase' + '<th>Status<th>Real time, s<th>User time, s<th>RSS, MB<th>'*len(rdirs) + '</tr>')
+    print('<tr><th>')
+
+    for dirname in rdirs:
+        for name in sorted(map(str, Path(dirname).glob('**/summary.tsv'))):
+            with open(name) as f:
+                coldata = []
+                cmdline = f.readline()
+                print('<th colspan="4"><span title="{}">{}</span><th>'.format(html.escape(cmdline, quote=True), html.escape(name)))
+                for line in f:
+                    line = line.split('\t')
+                    (q, utime, stime, maxrss, exitcode, elapsed) = line[:6]
+                    utime = float(utime)
+                    stime = float(stime)
+                    maxrss = int(maxrss)
+                    exitcode = int(exitcode)
+                    elapsed = int(elapsed)*1e-9
+                    if len(data):
+                        # assert data[0][len(coldata)][0] == q
+                        if data[0][len(coldata)][0] != q:
+                            pass
+                    coldata += [[dirname, q, elapsed, utime, stime, maxrss, exitcode]]
+                data += [coldata]
+    for i in range(len(data[0])):
+        q = data[0][i][1]
+        print('<tr><td>{}'.format(html.escape(q)), end='')
+        for c in range(len(data)):
+            (dirname, q, elapsed, utime, stime, maxrss, exitcode) = data[c][i]
+            outname = dirname + '/' + q + '-result.yson'
+            if exitcode < 0:
+                print('<td><span class="signal" title="{}">SIG</span>'.format(html.escape(signal.strsignal(-exitcode), quote=True)))
+            elif exitcode > 0:
+                print('<td><span class="errcode" title="{}">ERR</span>'.format(exitcode))
+            else:
+                print('<td><span class="ok">OK</span>')
+            print('<td class="tabnum">{:.1f}<td class="tabnum">{:.1f}<td class="tabnum">{:.1f}'.format(elapsed, utime, maxrss/1024))
+            if exitcode == 0:
+                try:
+                    valType = None
+                    valData = None
+                    with open(outname, 'rb') as f:
+                        for result in yson.list_fragments(yson.InputStream.from_file(f)):
+                            valType = result[0][b'Write'][0][b'Type']
+                            valData = result[0][b'Write'][0][b'Data']
+                            pass
+                    if c == 0:
+                        data[c][i] += [valType, valData]
+                        print('<td>')
+                    else:
+                        assert valType[0] == b'ListType'
+                        assert valType[1][0] == b'StructType'
+                        stypes = valType[1][1]
+                        ncols = len(stypes)
+                        refType = data[0][i][-2]
+                        refData = data[0][i][-1]
+                        refstypes = refType[1][1]
+                        assert ncols == len(refType[1][1]), 'Column number mismatch {} != {}'.format(ncols, len(refstypes))
+                        nrows = len(valData)
+                        assert nrows == len(refData), 'Row number mismatch {} != {}'.format(nrows, len(refData))
+                        for col in range(ncols):
+                            stype = stypes[col][1]
+                            isOptional = False
+                            if stype[0] == b'OptionalType':
+                                stype = stype[1]
+                                isOptional = True
+                            assert stype[0] == b'DataType'
+                            isDouble = stype[1] == b'Double'
+                            for row in range(nrows):
+                                val = valData[row][col]
+                                ref = refData[row][col]
+                                if isOptional:
+                                    if ref is None:
+                                        assert val is None, 'NULL != NOT NULL at {}, {}'.format(row, col)
+                                        continue
+                                    assert val is not None, 'NOT NULL != NULL at {}, {}'.format(row, col)
+                                    ref = ref[0]
+                                    val = val[0]
+                                if isDouble:
+                                    val = float(val)
+                                    ref = float(ref)
+                                    if math.isnan(val):
+                                        assert math.isnan(ref), '{} != {} at {}, {}'.format(val, ref, row, col)
+                                        continue
+                                    assert not math.isnan(ref), '{} != {} at {}, {}'.format(val, ref, row, col)
+                                    assert abs(val - ref) <= 1e-5*max(abs(val), abs(ref), 1), 'abs({} - {}) >= eps at {}, {}'.format(val, ref, row, col)
+                                else:
+                                    assert val == ref, '{} != {} type {} at {}, {}'.format(val, ref, stypes[col][1][1], row, col)
+                        print('<td class="ok">MATCH</td>')
+                except Exception:
+                    print('<td class="errcode">Comparison failed: ', traceback.format_exc())
+            else:
+                print('<td class="errcode">N/A')
+
+        print('</tr>')
+
+    print('</table>')
+    print('</html>')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/ydb/library/benchmarks/runner/result_compare/ya.make b/ydb/library/benchmarks/runner/result_compare/ya.make
@@ -0,0 +1,17 @@
+PY3_PROGRAM()
+
+OWNER(
+    yumkam7
+    g:yql
+    g:yql_ydb_core
+)
+
+PY_SRCS(
+    MAIN compare.py
+)
+
+PEERDIR(
+    library/python/cyson
+)
+
+END()
diff --git a/ydb/library/benchmarks/runner/result_convert/gen-report.py b/ydb/library/benchmarks/runner/result_convert/gen-report.py
@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+import sys
+import signal
+import html
+from pathlib import Path
+
+
+def main():
+    if len(sys.argv) < 2:
+        print('Usage: {} resultdir... >report.htm'.format(sys.argv[0]), file=sys.stderr)
+        sys.exit(1)
+    args = sys.argv[1:]
+    print('''
+<html><head><style>.signal { color: blue; } .errcode { color: red; } .ok { color: green; } .mismatch { color: yellow; } .tabnum { text-align: right; } </style></head>
+''')
+    print('<table border="1">')
+    print('<tr><th>Testcase' + '<th>Status<th>Real time, s<th>User time, s<th>RSS, MB'*len(args) + '</tr>')
+    data = []
+    print('<tr><th>')
+    for dirname in args:
+        for name in sorted(map(str, Path(dirname).glob('**/summary.tsv'))):
+            name = str(name)
+            with open(name) as f:
+                cmdline = f.readline()
+                print('<th colspan="4"><span title="{}">{}</span>'.format(html.escape(cmdline, quote=True), html.escape(name)))
+                coldata = []
+                for line in f:
+                    line = line.strip().split('\t')
+                    (q, utime, stime, maxrss, exitcode, elapsed) = line[:6]
+                    utime = float(utime)
+                    stime = float(stime)
+                    maxrss = int(maxrss)
+                    elapsed = int(elapsed)*1e-9
+                    exitcode = int(exitcode)
+                    coldata += [(q, elapsed, utime, stime, maxrss, exitcode)]
+                data += [coldata]
+
+    for i in range(len(data[0])):
+        q = data[0][i][0]
+        print('<tr><td>', q, end='')
+        for c in range(len(data)):
+            (q, elapsed, utime, stime, maxrss, exitcode) = data[c][i]
+            if exitcode < 0:
+                print('<td><span class="signal" title="{}">SIG</span>'.format(html.escape(signal.strsignal(-exitcode), quote=True)))
+            elif exitcode > 0:
+                print('<td><span class="errcode" title="{}">ERR</span>'.format(exitcode))
+            else:
+                print('<td><span class="ok">OK</span>')
+            print('<td class="tabnum">{:.1f}<td class="tabnum">{:.1f}<td class="tabnum">{:.1f}'.format(elapsed, utime, maxrss/1024))
+        print('</tr>')
+
+    print("</table>")
+    print("</html>")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/ydb/library/benchmarks/runner/result_convert/ya.make b/ydb/library/benchmarks/runner/result_convert/ya.make
@@ -0,0 +1,16 @@
+PY3_PROGRAM()
+
+OWNER(
+    yumkam7
+    g:yql
+    g:yql_ydb_core
+)
+
+PY_SRCS(
+    MAIN gen-report.py
+)
+
+PEERDIR(
+)
+
+END()
diff --git a/ydb/library/benchmarks/runner/run-tests.sh b/ydb/library/benchmarks/runner/run-tests.sh
@@ -0,0 +1,50 @@
+#!/bin/sh -ex
+: ${datasize=${1:-1}}
+: ${variant=${2:-h}}
+: ${script_path=${0%/*}}
+: ${ydb_path=$script_path/../../../..}
+(cd ${script_path} && $ydb_path/ya make --build relwithdebinfo)
+(cd ${ydb_path}/ydb/library/yql/tools/dqrun && $ydb_path/ya make --build relwithdebinfo)
+(cd ${ydb_path}/ydb/library/yql/udfs/common && $ydb_path/ya make --build relwithdebinfo datetime datetime2 string re2 set math unicode_base)
+(cd ${ydb_path}/ydb/library/benchmarks/gen_queries && $ydb_path/ya make --build relwithdebinfo)
+[ -d tpc/$variant/$datasize ] ||
+    ${ydb_path}/ydb/library/benchmarks/runner/download_files_${variant}_${datasize}.sh
+if [ x$variant = xds ]; then
+    xpragma="--pragma AnsiOptionalAs"
+else
+    xpragma=""
+fi
+[ -f ql-$datasize/$variant/bindings.json ] ||
+${ydb_path}/ydb/library/benchmarks/gen_queries/gen_queries \
+        --output ql-$datasize --variant ${variant} --syntax yql --dataset-size $datasize \
+        $xpragma \
+        #
+[ -f q-$datasize/$variant/bindings.json ] ||
+${ydb_path}/ydb/library/benchmarks/gen_queries/gen_queries \
+        --output q-$datasize --variant ${variant} --syntax yql --dataset-size $datasize \
+    --pragma dq.MaxTasksPerStage=1 \
+    --pragma dq.ComputeActorType="async" \
+    --pragma config.flags=LLVM_OFF \
+    $xpragma \
+# --pragma dq.UseFinalizeByKey=true \
+# --pragma dq.UseOOBTransport=true \
+#
+[ -f qs-$datasize/$variant/bindings.json ] ||
+${ydb_path}/ydb/library/benchmarks/gen_queries/gen_queries \
+        --output qs-$datasize --variant ${variant} --syntax yql --dataset-size $datasize \
+    --pragma dq.MaxTasksPerStage=1 \
+    --pragma config.flags=LLVM_OFF \
+    --pragma dq.ComputeActorType="async" \
+    --pragma dq.UseFinalizeByKey=true \
+    $xpragma \
+# --pragma dq.UseOOBTransport=true \
+#
+outdir=results-`date -u +%Y%m%dT%H%M%S`-$datasize
+if false; then
+echo LLVM && \
+command time ${script_path}/runner/runner ql-$datasize/${variant} ql-$datasize/bindings.json $outdir ${ydb_path}/ydb/library/yql/tools/dqrun/dqrun -s --fs-cfg ${ydb_path}/ydb/library/yql/tools/dqrun/examples/fs.conf --gateways-cfg $script_path/runner/test-gateways.conf --udfs-dir ${ydb_path}/ydb/library/yql/udfs/common/
+fi
+echo NO LLVM && \
+command time ${script_path}/runner/runner q-$datasize/${variant} q-$datasize/${variant}/bindings.json $outdir ${ydb_path}/ydb/library/yql/tools/dqrun/dqrun -s --fs-cfg ${ydb_path}/ydb/library/yql/tools/dqrun/examples/fs.conf --gateways-cfg $script_path/runner/test-gateways.conf --udfs-dir ${ydb_path}/ydb/library/yql/udfs/common/
+echo Spilling && \
+command time ${script_path}/runner/runner qs-$datasize/${variant} qs-$datasize/${variant}/bindings.json $outdir ${ydb_path}/ydb/library/yql/tools/dqrun/dqrun -s --enable-spilling --fs-cfg ${ydb_path}/ydb/library/yql/tools/dqrun/examples/fs.conf --gateways-cfg $script_path/runner/test-gateways.conf --udfs-dir ${ydb_path}/ydb/library/yql/udfs/common/
diff --git a/ydb/library/benchmarks/runner/runner/runner.py b/ydb/library/benchmarks/runner/runner/runner.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+import os
+import sys
+from pathlib import Path
+import re
+import datetime
+
+try:
+    from time import clock_gettime_ns, CLOCK_MONOTONIC
+
+    def time_ns():
+
+        return clock_gettime_ns(CLOCK_MONOTONIC)
+
+except Exception:
+    from time import time_ns
+
+
+RE_DIGITS = re.compile(r'([0-9]+)')
+
+
+def run(argv, out, err):
+
+    start_time = time_ns()
+    pid = os.posix_spawn(argv[0], argv, {}, file_actions=(
+        (os.POSIX_SPAWN_OPEN, 1, out, os.O_WRONLY | os.O_CREAT, 0o666),
+        (os.POSIX_SPAWN_OPEN, 2, err, os.O_WRONLY | os.O_CREAT, 0o666),
+        ))
+    (pid, status, rusage) = os.wait4(pid, 0)
+    elapsed = time_ns()
+    elapsed -= start_time
+    exitcode = os.waitstatus_to_exitcode(status)
+    return exitcode, rusage, elapsed
+
+
+def main():
+
+    qdir = sys.argv[1] or 'q/scalar'
+    bindings = sys.argv[2] or 'bindings.json'
+    outdir = sys.argv[3] or "result-{:%Y%m%dT%H%M%S}".format(datetime.datetime.now())
+    argv = sys.argv[4:]
+    assert len(argv)
+    querydir = Path(qdir)
+    os.makedirs(outdir + '/' + qdir, exist_ok=True)
+    with open(outdir + '/' + qdir + "/summary.tsv", "w") as outf:
+        print(' '.join(argv + ['-p', qdir, '--bindings-file', bindings]), file=outf)
+        for query in sorted(querydir.glob('**/*.sql'), key=lambda x: tuple(map(lambda y: int(y) if re.match(RE_DIGITS, y) else y, re.split(RE_DIGITS, str(x))))):
+            q = str(query)
+            print(q, end='\t', file=outf)
+            name = outdir + '/' + q
+            outname = name + '-result.yson'
+            exitcode, rusage, elapsed = run(
+                argv + [
+                    '--result-file', outname,
+                    '--bindings-file', bindings,
+                    '--plan-file', name + '-plan.yson',
+                    '--err-file', name + '-err.txt',
+                    '--expr-file', name + '-expr.txt',
+                    '-p', q
+                ],
+                name + '-stdout.txt',
+                name + '-stderr.txt')
+            print(rusage.ru_utime, end='\t', file=outf)
+            print(rusage.ru_stime, end='\t', file=outf)
+            print(rusage.ru_maxrss, end='\t', file=outf)
+            print(exitcode, end='\t', file=outf)
+            print(elapsed, end='\t', file=outf)
+            print(rusage.ru_minflt, end='\t', file=outf)
+            print(rusage.ru_majflt, end='\t', file=outf)
+            print(rusage.ru_inblock, end='\t', file=outf)
+            print(rusage.ru_oublock, end='\t', file=outf)
+            print(rusage.ru_nvcsw, end='\t', file=outf)
+            print(rusage.ru_nivcsw, end='\t', file=outf)
+            # resource.struct_rusage(ru_utime=7.919329, ru_stime=5.22704,
+            #   ru_maxrss=639600, ru_ixrss=0, ru_idrss=0, ru_isrss=0,
+            #   ru_minflt=135127, ru_majflt=0, ru_nswap=0, ru_inblock=0,
+            #   ru_oublock=48, ru_msgsnd=0, ru_msgrcv=0, ru_nsignals=0,
+            #   ru_nvcsw=57452, ru_nivcsw=273
+            # )
+            print(file=outf)
+            outf.flush()
+
+
+if __name__ == "__main__":
+    main()