Add support for GYB in benchmarks

benchmark/CMakeLists.txt

@@ -44,7 +44,9 @@ set(SWIFT_BENCH_MODULES
     single-source/DictionaryLiteral
     single-source/DictionaryRemove
     single-source/DictionarySwap
+    single-source/DropFirst
     single-source/DropLast
+    single-source/DropWhile
     single-source/ErrorHandling
     single-source/ExistentialPerformance
     single-source/Fibonacci
@@ -79,6 +81,8 @@ set(SWIFT_BENCH_MODULES
     single-source/PolymorphicCalls
     single-source/PopFront
     single-source/PopFrontGeneric
+    single-source/Prefix
+    single-source/PrefixWhile
     single-source/Prims
     single-source/ProtocolDispatch
     single-source/ProtocolDispatch2

benchmark/scripts/generate_harness/generate_harness.py

@@ -12,43 +12,75 @@
 #
 # ===---------------------------------------------------------------------===//
 
-# Generate CMakeLists.txt and utils/main.swift from templates.
+# Generate boilerplate, CMakeLists.txt and utils/main.swift from templates.
 
 from __future__ import print_function
 
-import glob
+import argparse
 import os
 import re
+import subprocess
 
 import jinja2
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 perf_dir = os.path.realpath(os.path.join(script_dir, '../..'))
+gyb = os.path.realpath(os.path.join(perf_dir, '../utils/gyb'))
 single_source_dir = os.path.join(perf_dir, 'single-source')
 multi_source_dir = os.path.join(perf_dir, 'multi-source')
+parser = argparse.ArgumentParser()
+parser.add_argument("--output-dir",
+                    help="Output directory (for validation test)",
+                    default=perf_dir)
+args = parser.parse_args()
+output_dir = args.output_dir
 
 template_map = {
-    'CMakeLists.txt_template': os.path.join(perf_dir, 'CMakeLists.txt'),
-    'main.swift_template': os.path.join(perf_dir, 'utils/main.swift')
+    'CMakeLists.txt_template': os.path.join(output_dir, 'CMakeLists.txt'),
+    'main.swift_template': os.path.join(output_dir, 'utils/main.swift')
 }
 ignored_run_funcs = ["Ackermann", "Fibonacci"]
 
 template_loader = jinja2.FileSystemLoader(searchpath="/")
 template_env = jinja2.Environment(loader=template_loader, trim_blocks=True,
                                   lstrip_blocks=True)
 
+
+def all_files(directory, extension):  # matching: [directory]/**/*[extension]
+    return [
+        os.path.join(root, f)
+        for root, _, files in os.walk(directory)
+        for f in files if f.endswith(extension)
+    ]
+
+
+def will_write(filename):  # ensure path to file exists before writing
+    print(filename)
+    output_path = os.path.split(filename)[0]
+    if not os.path.exists(output_path):
+        os.makedirs(output_path)
+
+
 if __name__ == '__main__':
+    # Generate Your Boilerplate
+    gyb_files = all_files(perf_dir, '.gyb')
+    for f in gyb_files:
+        relative_path = os.path.relpath(f[:-4], perf_dir)
+        out_file = os.path.join(output_dir, relative_path)
+        will_write(out_file)
+        subprocess.call([gyb, '--line-directive', '', '-o', out_file, f])
+
     # CMakeList single-source
-    test_files = glob.glob(os.path.join(single_source_dir, '*.swift'))
+    test_files = all_files(single_source_dir, '.swift')
     tests = sorted(os.path.basename(x).split('.')[0] for x in test_files)
 
     # CMakeList multi-source
     class MultiSourceBench(object):
-
         def __init__(self, path):
             self.name = os.path.basename(path)
             self.files = [x for x in os.listdir(path)
                           if x.endswith('.swift')]
+
     if os.path.isdir(multi_source_dir):
         multisource_benches = [
             MultiSourceBench(os.path.join(multi_source_dir, x))
@@ -67,26 +99,19 @@ def get_run_funcs(filepath):
         matches = re.findall(r'func run_(.*?)\(', content)
         return filter(lambda x: x not in ignored_run_funcs, matches)
 
-    def find_run_funcs(dirs):
-        ret_run_funcs = []
-        for d in dirs:
-            for root, _, files in os.walk(d):
-                for name in filter(lambda x: x.endswith('.swift'), files):
-                    run_funcs = get_run_funcs(os.path.join(root, name))
-                    ret_run_funcs.extend(run_funcs)
-        return ret_run_funcs
-    run_funcs = sorted(
-        [(x, x)
-         for x in find_run_funcs([single_source_dir, multi_source_dir])],
-        key=lambda x: x[0]
-    )
+    def find_run_funcs():
+        swift_files = all_files(perf_dir, '.swift')
+        return [func for f in swift_files for func in get_run_funcs(f)]
+
+    run_funcs = [(f, f) for f in sorted(find_run_funcs())]
 
     # Replace originals with files generated from templates
     for template_file in template_map:
         template_path = os.path.join(script_dir, template_file)
         template = template_env.get_template(template_path)
-        print(template_map[template_file])
-        open(template_map[template_file], 'w').write(
+        out_file = template_map[template_file]
+        will_write(out_file)
+        open(out_file, 'w').write(
             template.render(tests=tests,
                             multisource_benches=multisource_benches,
                             imports=imports,

benchmark/scripts/generate_harness/test_generate_harness.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+# This script is invoked by `lit` on all smoke test runs from the
+# validation-test/benchmark/generate-harness.test-sh.
+# It ensures that the files checked in the benchmark suite that are generated
+# from templates always match what would be regenerated if one
+# re-ran the relevant scripts. This is to catch accidental manual edits.
+
+SWIFT_SRC_DIR="$1"
+BENCHMARK_DIR="${SWIFT_SRC_DIR}/benchmark"
+SCRIPT_DIR="${BENCHMARK_DIR}/scripts"
+TEMP_DIR="$2"
+
+"${SCRIPT_DIR}/generate_harness/generate_harness.py" "--output-dir=${TEMP_DIR}"
+for f in $(cd "${TEMP_DIR}" && find ./ -type f); do
+    diff "${TEMP_DIR}/${f}" "${BENCHMARK_DIR}/${f}"
+    if [[ $? -ne 0 ]]; then
+       exit 1
+    fi
+done

benchmark/single-source/DropFirst.swift

@@ -0,0 +1,193 @@
+//===--- DropFirst.swift --------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//
+
+////////////////////////////////////////////////////////////////////////////////
+// WARNING: This file is manually generated from .gyb template and should not
+// be directly modified. Instead, make changes to DropFirst.swift.gyb and run
+// scripts/generate_harness/generate_harness.py to regenerate this file.
+////////////////////////////////////////////////////////////////////////////////
+
+import TestsUtils
+
+let sequenceCount = 4096
+let dropCount = 1024
+let suffixCount = sequenceCount - dropCount
+let sumCount = suffixCount * (2 * sequenceCount - suffixCount - 1) / 2
+
+@inline(never)
+public func run_DropFirstCountableRange(_ N: Int) {
+  let s = 0..<sequenceCount
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstCountableRange: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstSequence(_ N: Int) {
+  let s = sequence(first: 0) { $0 < sequenceCount - 1 ? $0 &+ 1 : nil }
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstSequence: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstAnySequence(_ N: Int) {
+  let s = AnySequence(sequence(first: 0) { $0 < sequenceCount - 1 ? $0 &+ 1 : nil })
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstAnySequence: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstAnySeqCntRange(_ N: Int) {
+  let s = AnySequence(0..<sequenceCount)
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstAnySeqCntRange: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstAnySeqCRangeIter(_ N: Int) {
+  let s = AnySequence((0..<sequenceCount).makeIterator())
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstAnySeqCRangeIter: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstAnyCollection(_ N: Int) {
+  let s = AnyCollection(0..<sequenceCount)
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstAnyCollection: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstArray(_ N: Int) {
+  let s = Array(0..<sequenceCount)
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstArray: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstCountableRangeLazy(_ N: Int) {
+  let s = (0..<sequenceCount).lazy
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstCountableRangeLazy: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstSequenceLazy(_ N: Int) {
+  let s = (sequence(first: 0) { $0 < sequenceCount - 1 ? $0 &+ 1 : nil }).lazy
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstSequenceLazy: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstAnySequenceLazy(_ N: Int) {
+  let s = (AnySequence(sequence(first: 0) { $0 < sequenceCount - 1 ? $0 &+ 1 : nil })).lazy
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstAnySequenceLazy: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstAnySeqCntRangeLazy(_ N: Int) {
+  let s = (AnySequence(0..<sequenceCount)).lazy
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstAnySeqCntRangeLazy: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstAnySeqCRangeIterLazy(_ N: Int) {
+  let s = (AnySequence((0..<sequenceCount).makeIterator())).lazy
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstAnySeqCRangeIterLazy: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstAnyCollectionLazy(_ N: Int) {
+  let s = (AnyCollection(0..<sequenceCount)).lazy
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstAnyCollectionLazy: \(result) != \(sumCount)")
+  }
+}
+@inline(never)
+public func run_DropFirstArrayLazy(_ N: Int) {
+  let s = (Array(0..<sequenceCount)).lazy
+  for _ in 1...20*N {
+    var result = 0
+    for element in s.dropFirst(dropCount) {
+      result += element
+    }
+    CheckResults(result == sumCount,
+      "IncorrectResults in DropFirstArrayLazy: \(result) != \(sumCount)")
+  }
+}