feat: add automation script to test backends

backends/dart_conduit/pubspec.lock

@@ -56,63 +56,63 @@ packages:
       name: conduit
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.5"
+    version: "3.2.12"
   conduit_codable:
     dependency: transitive
     description:
       name: conduit_codable
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.4"
+    version: "3.2.11"
   conduit_common:
     dependency: transitive
     description:
       name: conduit_common
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.4"
+    version: "3.2.11"
   conduit_config:
     dependency: transitive
     description:
       name: conduit_config
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.4"
+    version: "3.2.11"
   conduit_isolate_exec:
     dependency: transitive
     description:
       name: conduit_isolate_exec
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.4"
+    version: "3.2.11"
   conduit_open_api:
     dependency: transitive
     description:
       name: conduit_open_api
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.4"
+    version: "3.2.11"
   conduit_password_hash:
     dependency: transitive
     description:
       name: conduit_password_hash
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.4"
+    version: "3.2.11"
   conduit_runtime:
     dependency: transitive
     description:
       name: conduit_runtime
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.4"
+    version: "3.2.11"
   conduit_test:
     dependency: "direct dev"
     description:
       name: conduit_test
       url: "https://pub.dartlang.org"
     source: hosted
-    version: "3.2.4"
+    version: "3.2.11"
   convert:
     dependency: transitive
     description:

backends/dart_conduit/run.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo 'Getting dependencies...'
+dart pub get
+
+echo 'Activating conduit'
+dart pub global activate conduit
+
+echo 'Running conduit'
+conduit serve -a 127.0.0.1 -p 8080

backends/dart_frog_backend/pubspec.yaml

@@ -8,7 +8,7 @@ environment:
 
 dependencies:
   compute: ^1.0.2
-  dart_frog: ^0.2.0
+  dart_frog: ^0.3.1
   shelf_multipart: ^1.0.0
 
 dev_dependencies:

backends/dart_frog_backend/setup.sh → backends/dart_frog_backend/run.sh

@@ -1,4 +1,4 @@
-#!bin/sh
+#!/bin/sh
 
 echo 'Getting dependencies...'
 dart pub get
@@ -7,4 +7,7 @@ echo 'Activating dart_frog_cli'
 dart pub global activate dart_frog_cli
 
 echo 'Build server'
-dart_frog_cli build
+dart_frog build
+
+echo 'Run server'
+dart build/bin/server.dart

backends/dart_minerva/run.sh

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+echo 'Getting dependencies...'
+dart pub get
+
+echo 'Activating Minerva...'
+dart pub global activate minerva
+
+echo 'Running Minerva...'
+minerva run

backends/dart_shelf/run.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+echo 'Getting dependencies...'
+dart pub get
+
+echo 'Running server...'
+dart bin/server.dart

backends/node_express/run.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+echo 'Getting dependencies...'
+npm install
+
+echo 'Running server...'
+npm start

backends/py_flask/setup.sh → backends/py_flask/run.sh

@@ -1,4 +1,7 @@
-#!bin/sh
+#!/bin/sh
 
 echo 'Getting dependencies...'
 pip install -r requirements.txt
+
+echo 'Running server...'
+python serve.py

benchmark/.gitignore

@@ -4,3 +4,4 @@
 
 # Conventional directory for build output.
 build/
+results/

benchmark/bin/benchmark.dart

@@ -1,10 +1,13 @@
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:benchmark/benchmark.dart' as benchmark;
 
 const interation = 20;
 
 Future<int> main(List<String> arguments) async {
+  final name = arguments.length == 1 ? arguments[0] : "none";
+
   final List<double> rttGet = [],
       rttPost = [],
       rttGetParallel = [],
@@ -50,5 +53,27 @@ Future<int> main(List<String> arguments) async {
   print("\nJSON parsing speed");
   print("Parse JSON: ${parseJsonTime.average()}ms");
 
+  saveResults(
+    {
+      "get_rtt": rttGet.average(),
+      "post_rtt": rttPost.average(),
+      "get_rtt_parallel": rttGetParallel.average(),
+      "post_rtt_parallel": rttPostParallel.average(),
+      "send_file": sendFileTime.average(),
+      "parse_json": parseJsonTime.average(),
+      "file_size": fileSize,
+    },
+    name,
+  );
+
   return 0;
 }
+
+void saveResults(Map<String, num> result, String name) {
+  final file = File("./results/$name.json");
+  if (file.existsSync()) {
+    file.deleteSync();
+  }
+  file.createSync(recursive: true);
+  file.writeAsStringSync(jsonEncode(result));
+}

generate-report.sh

@@ -0,0 +1,81 @@
+#!/bin/sh
+
+set -eu;
+
+# Set up dart pub environment
+export PATH="$PATH":"$HOME/.pub-cache/bin";
+
+(
+	cd "benchmark";
+	dart pub get;
+)
+
+cd "$(dirname "$0")";
+
+TEST_BACKENDS=$(ls backends);
+if [ "$#" -gt 0 ]; then
+	TEST_BACKENDS="$*";
+fi
+
+for backend in ${TEST_BACKENDS}; do
+	if [ "${backend}" = "dart_conduit" ]; then
+		echo "skipping ${backend} due to issues";
+		continue;
+	fi
+
+	echo "testing ${backend}";
+
+	(
+		cd "backends/${backend}";
+		chmod +x ./run.sh;
+		./run.sh;
+	) | sed "s/^/  /" &
+
+	# wait for server to start
+	while ! curl -q "http://127.0.0.1:8080" > /dev/null 2>&1; do
+		sleep 3;
+	done
+
+	# run tests
+	echo "running tests - ${backend}";
+	(
+		cd "benchmark";
+		dart run benchmark ${backend};
+	)
+
+	# kill children processes
+	echo "killing ${backend} process";
+	children_pids="";
+	for pid in /proc/*; do
+		case "${pid}" in
+			/proc/*[!0-9]* | "/proc/$$")
+				continue;
+				;;
+			*)
+				if grep -P "^\d+ \(.*\) [RSDZTW] \d+ $$ " "${pid}/stat" > /dev/null; then
+					children_pids="${children_pids} ${pid#/proc/}";
+				fi
+				;;
+		esac
+	done
+
+	children_running="true";
+	while [ "${children_running}" = "true" ]; do
+		children_running="false";
+		for pid in ${children_pids}; do
+			if ps -p "${pid}" > /dev/null; then
+				children_running="true";
+				echo "killing ${backend} with pid ${pid}";
+				kill "${pid}";
+			fi
+		done
+
+		sleep 1;
+	done
+
+	# wait for server to stop
+	# while curl -q "http://127.0.0.1:8080" > /dev/null 2>&1; do
+	# 	sleep 3;
+	# done
+	echo;
+done