lit.cfg

import lit.formats
import lit.util
import lit
import os, glob, re
import shlex
import pipes
from lit.formats import FileBasedTest
from lit.TestRunner import executeScript, executeScriptInternal, parseIntegratedTestScriptCommands, getDefaultSubstitutions, applySubstitutions, getTempPaths
from lit import Test
from lit.util import to_bytes, to_string
try:
    from shlex import quote # python 3.3 and above
except:
    from pipes import quote # python 3.2 and earlier

def parseBenchmarkScript(test):
    """Scan a llvm-testsuite like benchmark .test script."""
    def parseShellCommand(script, ln):
        # Trim trailing whitespace.
        ln = ln.rstrip()

        # Collapse lines with trailing '\\'.
        if script and script[-1][-1] == '\\':
            script[-1] = script[-1][:-1] + ln
        else:
            script.append(ln)

    # Collect the test lines from the script.
    sourcepath = test.getSourcePath()
    runscript = []
    verifyscript = []
    keywords = ['RUN:', 'VERIFY:']
    for line_number, command_type, ln in \
            parseIntegratedTestScriptCommands(sourcepath, keywords):
        if command_type == 'RUN':
            parseShellCommand(runscript, ln)
        elif command_type == 'VERIFY':
            parseShellCommand(verifyscript, ln)
        else:
            raise ValueError("unknown script command type: %r" % (
                    command_type,))

    # Verify the script contains a run line.
    if runscript == []:
        return lit.Test.Result(Test.UNRESOLVED, "Test has no RUN: line!")

    # Check for unterminated run lines.
    for script in runscript, verifyscript:
        if script and script[-1][-1] == '\\':
            return lit.Test.Result(Test.UNRESOLVED,
                                   "Test has unterminated RUN/VERIFY lines (with '\\')")

    return runscript,verifyscript

def getUserTimeFromTimeOutput(f):
    with open(f) as fd:
        l = [l for l in fd.readlines()
             if l.startswith('user')]
    assert len(l) == 1

    m = re.match(r'user\s+([0-9.]+)', l[0])
    return float(m.group(1))

def collectCompileTime(test):
    # TODO: This is not correct yet as the directory may contain .o.time files
    # of multiple benchmarks in the case of SingleSource tests.
    compile_time = 0.0
    basepath = os.path.dirname(test.getFilePath())
    for path, subdirs, files in os.walk(basepath):
        for file in files:
            if file.endswith('.o.time'):
                compile_time += getUserTimeFromTimeOutput(os.path.join(path, file))
    return compile_time

def runScript(test, litConfig, script, tmpBase, useExternalSh = True):
    execdir = os.path.dirname(test.getExecPath())
    if useExternalSh:
        res = executeScript(test, litConfig, tmpBase, script, execdir)
    else:
        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
    return res

class TestSuiteTest(FileBasedTest):
    def __init__(self):
        super(TestSuiteTest, self).__init__()
        
    def execute(self, test, litConfig):
        if test.config.unsupported:
            return (Test.UNSUPPORTED, 'Test is unsupported')

        # Parse benchmark script
        res = parseBenchmarkScript(test)
        if isinstance(res, lit.Test.Result):
            return res
        if litConfig.noExecute:
            return lit.Test.Result(Test.PASS)
        runscript, verifyscript = res

        tmpDir, tmpBase = getTempPaths(test)
        outfile = tmpBase + ".out"
        substitutions = getDefaultSubstitutions(test, tmpDir, tmpBase)
        substitutions += [('%o', outfile)]
        runscript = applySubstitutions(runscript, substitutions)
        verifyscript = applySubstitutions(verifyscript, substitutions)

        # Prepend runscript with RunSafely and timeit stuff
        def prependRunSafely(line):
            # Search for "< INPUTFILE" in the line and use that for stdin
            stdin = "/dev/null"
            commandline = shlex.split(line)
            for i in range(len(commandline)):
                if commandline[i] == "<" and i+1 < len(commandline):
                    stdin = commandline[i+1]
                    del commandline[i+1]
                    del commandline[i]
                    break
            timeit = config.test_source_root + "/tools/timeit"
            timeout = "7200"
            runsafely_prefix = ["%s/RunSafely.sh" % config.test_suite_root,
                                "-t", timeit, timeout, stdin, outfile]

            line = " ".join(map(quote, runsafely_prefix + commandline))
            return line
        runscript = map(prependRunSafely, runscript)

        # Create the output directory if it does not already exist.
        lit.util.mkdir_p(os.path.dirname(tmpBase))

        # Execute runscript (the "RUN:" part)
        output = ""
        n_runs = 1
        runtimes = []
        for n in range(n_runs):
            res = runScript(test, litConfig, runscript, tmpBase)
            if isinstance(res, lit.Test.Result):
                return res

            output += "\n" + "\n".join(runscript)

            out, err, exitCode = res
            if exitCode == Test.FAIL:
                # Only show command output in case of errors
                output += "\n" + out
                output += "\n" + err
                return lit.Test.Result(Test.FAIL, output)

            timefile = "%s.time" % (outfile,)
            runtime = getUserTimeFromTimeOutput(timefile)
            runtimes.append(runtime)

        # Run verification script (the "VERIFY:" part)
        if len(verifyscript) > 0:
            res = runScript(test, litConfig, verifyscript, tmpBase)
            if isinstance(res, lit.Test.Result):
                return res
            out, err, exitCode = res

            output += "\n" + "\n".join(verifyscript)
            if exitCode != 0:
                output += "\n" + out
                output += "\n" + err
                return lit.Test.Result(Test.FAIL, output)

        compile_time = collectCompileTime(test)

        result = lit.Test.Result(Test.PASS, output)
        result.addMetric('exec_time', lit.Test.toMetricValue(runtimes[0]))
        result.addMetric('compile_time', lit.Test.toMetricValue(compile_time))
        return result

config.name = 'test-suite'
config.test_format = TestSuiteTest()
config.suffixes = ['.test']
config.excludes = ['ABI-Testsuite']