diff --git a/.gitignore b/.gitignore index 71cbabe..277a336 100644 --- a/.gitignore +++ b/.gitignore @@ -1,22 +1,35 @@ -#*# +*.Rout *.beam *.class *.hi *.o *.plt +*.pyc *~ .* -*.pyc -*.Rout -data/ +\#*# +competition/cljwsdemo/target/ +competition/log/*.log +competition/node_modules +competition/wsdemo-snap +competition/wsdemo-go-gonet +competition/pypy-1.9 +competition/supervisord.pid +competition/wsdemo-go-gonet +competition/play-2.0.1* +competition/wsdemo-playframework-2/project/target/ +competition/wsdemo-playframework-2/target/ +competition/wsdemo-playframework-2/wsdemo-1.0-SNAPSHOT/ +competition/cowboy-*/ +data deps/ dist/ ebin/ +node_modules results/ -stats/stat_results/ stats/*.Rout stats/*.csv -competition/log/*.log -competition/wsdemo-snap -competition/node_modules -competition/cljwsdemo/target/ +stats/db +stats/stat_results/ +Vagrantfile +erl_crash.dump diff --git a/AUTHORS b/AUTHORS new file mode 100644 index 0000000..e72db2d --- /dev/null +++ b/AUTHORS @@ -0,0 +1,28 @@ +I would like to take this opportunity to thank those that wrote the +individual Websocket projects and wsdemo implementations: + +## wsdemo implementations + + * Alex Efros - For the Perl/EV implementation + * Alexander Glyzov - For the Twisted, Tornado and Gevent Python implementations + * Einar Otto Stangvik - For the Clustered ws/node.js implementation + * Oliver Charles - For the Haskell/Snap implementation + * Ravi Kotecha - For the Clojure/Aleph implementation + * maqdev - For the Play Frameware implementation + +## Websocket projects + + * Corbin Simpson - For the the txWS project + * Einar Otto Stangvik - For the ws node.js module + * Jeffery Gelens - For the gevent-websocket project + * Joe Walnes - For the Webbit project + * Loïc Hoguin and Nine Nines - For the Erlang cowboy project + * The Snap team - For the Haskell Snap framework + * The Tornado team - For the tornado.websocket module + * The go team - For the net/websockets code + * Viacheslav Tykhanovskyi - For the Protocol::Websocket project + * Zach Tellman - For the Clojure Aleph project + +## Other contributions + + * Jesper Louis Andersen - For the stats/ code diff --git a/Makefile b/Makefile index 5107cef..5af230a 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ build: ./rebar get-deps compile + $(MAKE) -C competition build report: ./bin/compile_all_stats.sh diff --git a/README.md b/README.md index 902ffa5..e16e6dc 100644 --- a/README.md +++ b/README.md @@ -1,37 +1,201 @@ # wsdemo -A Cowboy Websocket demo +A Websocket competition. Will your favorite platform come out on top? +If you think your platform of choice was poorly represented, "submit a +pull request™" -This demo is intended to follow along with -the [Million-user Comet Application](http://www.metabrew.com/article/a-million-user-comet-application-with-mochiweb-part-1) +## How do you include a contestant? -## Demo +Just create the "Hello, World!" of socket servers, an echo server that +speaks WebSocket version 13 (13 is the only version my client supports). -Compile: +First write your implementation in place it in the `competition/` +directory. - ./rebar get-deps compile +Added any instructions needed to install dependencies in +`configure_ubuntu-12.04.sh`. Add any build instructions to +`competition/Makefile` if your project needs to be compiled +wsdemo_bench utilizes [supervisord](http://supervisord.org/) to start +and stop the servers while running the tests. In order to include +your server in the test you need to create a supervisord ini file in +`competition/servers`. + [program:%(server_name)s] + command=%(command)s + autostart=false + autorestart=false + startretries=0 + stopasgroup=true + killasgroup=true + +The server name must follow the following format: -Follow the "Tuning the Linux Kernel for many tcp connections" instructions in Part One of the -[Million-user Comet Application](http://www.metabrew.com/article/a-million-user-comet-application-with-mochiweb-part-1) -to increase your available connection counts. + {language}-{platform} -Start the server: +For instance if you wrote an echo server using `bash` and `nc` the +server name would be `bash-nc`. You can add additional demarcations +if needed. For instance the Python tornado example has a single +threaded version and a multiprocessor version which go by the name +`python-tornado-1` and `python-tornado-N`. - sudo bash - ulimit -n 999999 - erl -pa ebin deps/*/ebin -s wsdemo +Next, you need to add your server to the server list in +`wsdemo_bench.app.src` configuration file. -Start floodtest: +Finally, add yourself to the AUTHORS file; you've earned it. - sudo bash - ulimit -n 999999 - ./runtest results/erlang.bin 300 10000 +## Running the benchmark -Compile the results: +It is best to run the benchmark client on a separate machine than the +servers. - ./compile_stats results/erlang.bin +Using two brand new Ubuntu 12.04 servers, run the +`configure_ubuntu-12.04.sh` script on each. This script will destructively +alter each server so use caution and review the script before running +it. -I am planning on rerunning these benchmarks with the new stats collecting code using 64bit m1.large instances. -The [previous results](https://github.com/ericmoritz/wsdemo/blob/results-v1/results.md) are still available +If you want to use a different OS, use `configure_ubuntu-12.04.sh` as a +template for configuring your system of choice. + +Next compile the code on both machines: + + make + +To run a single benchmark on a server do the following: + + ./runtest db_path duration hostname port client_count + +For instance for a benchmark of 192.168.1.5:8000 of 1000 clients lasting 60 +seconds with with the data stored at `data/myserver` the command would +be: + + ./runtest data/myserver 60 192.168.1.5 8000 1000 + + +## Running the entire suite on your own servers + +There are two components in wsdemo_bench. The first component is +`supervisord` + +wsdemo_bench communicates with Supervisord to start and stop each +server before each benchmark. + +On the machine that you are running the servers on do the following: + + sudo bash + ulimit -n 999999 + cd competition + supervisord + +You can monitor supervisord by using the `supervisorctl` command: + + competition/ $ supervisorctl status + + +Next, on the client machine create a `config/small-scale.config` file: + + [ + {sasl, [ + {sasl_error_logger, {file, LogFile}}]}, + + {wsdemo_bench, [ + {host, Ip}, + {port, 8000}, + {db_root, DbRoot}, + {clients, 100}, + {seconds, 10}, + {supervisord, {ServerHost, 9001}}]}]. + + +Replace `LogFile`, `Ip`, `DbRoot` and `ServerHost` with the correct +values. Ensure that both the path to `LogFile` and the path to +`DbRoot` exist or the test will crash. + +This configuration file describe a test using 100 clients for +10 seconds each test. We will use this smaller test to ensure that all +the servers start and stop properly. + +To run this benchmark, do the following: + + sudo bash + ulimit -n 999999 + ./bin/run_all_tests.sh -config config/small-scale.config + +If all goes well, you should see the suite running in front of your +eyes. Let that run for the entire test. It should run for 12 minutes +or so. + +If any of the servers crashes you can do the following to diagnose the cause: + + competition/ $ supervisorctl tail $SERVER_NAME stderr + +After the small scale test, it is time to run the full scale test: + + [ + {sasl, [ + {sasl_error_logger, {file, LogFile}}]}, + + {wsdemo_bench, [ + {host, Ip}, + {port, 8000}, + {db_root, DbRoot}, + {supervisord, {ServerHost, 9001}}]}]. + +## Suite config + +Here is the full spec for the wsdemo_bench config: + + [ + {host, Host :: string()}, + {port, Port :: integer()}, + {db_root, DbRoot :: directory()}, + {supervisord, {Host, 9001}}, + {clients, number()}, % defaults to 10000 + {seconds, number()}, % defaults to 600 + {servers, [ServerName1::string(), ..., ServerNameN::string()]} % defaults to full suite + ] + + +## Exporting the data + +The resulting `leveldb` databases will be placed in your configured +`db_root`. The events are stored as binary Erlang terms in the +database so you will need to export the events to use them. + +There are two scripts to do that. `./bin/compile_all_stats.sh` and +`./bin/convert_all_stats.sh` + +`./bin/compile_all_stats.sh` creates three `.csv` tables: + + * counts.csv - the sums of all the events + * handshake_times.csv - timestamp, elapsed usecs for each handshake + * message_latencies.csv - timestamp, elapsed usecs for each message + +`./bin/convert_all_stats.sh` dumps the raw events as a `events.csv` table. + +The events data has the following fields: + + timestamp, type, client_id, event_key, event_data + + * timestamp - The timestamp of the event + * type - the server type + * client_id - A string that identifies the client + * event_key - The event type + * event_data - event specific data + +The `event_data` is used to pair `ws_init` and `ws_onopen` events to +calculate the elapsed handshake time and the `event_data` is used to +pair `send_message` events to `recv_message` events to calculate the +elapsed message time. The `event_data` for `'EXIT'` events take the +format of `{pid(), Reason :: any()}`. These are some of the expected +reasons for a client crash: + + connection_timeout - The client could not establish a connection + with the server in under 2 seconds + normal - The server closed the connection. This should never + happen for this benchmark + +Any reason other than these are an unexpected error and should be +[reported](https://github.com/ericmoritz/wsdemo/issues) as an issue. +If you see a `{error, Reason}` you should probably file a bug report +as well unless you can determine it is a server issue. diff --git a/bin/compile_all_stats.sh b/bin/compile_all_stats.sh index 0631203..75347c6 100755 --- a/bin/compile_all_stats.sh +++ b/bin/compile_all_stats.sh @@ -1,5 +1,5 @@ -for db_path in data/*; do +for db_path in "$1"/*; do server_type=`basename $db_path` echo "Compiling tables for $server_type" ./bin/compile_stats $server_type $db_path diff --git a/bin/compile_stats b/bin/compile_stats index b88cdb2..e130941 100755 --- a/bin/compile_stats +++ b/bin/compile_stats @@ -17,14 +17,14 @@ counts=#counts{}}). write_stats_head(FH) -> - io:format(FH, "type, clients, handshakes, connection_timeouts, messages_sent, messages_recv, crashes~n", []). + io:format(FH, "type,clients,handshakes,connection_timeouts,messages_sent,messages_recv,crashes~n", []). write_stats(FH, #state{counts=Counts, type=Type}) -> io:format(FH, "\"~s\"", [Type]), lists:foreach(fun(Field) -> Val = element(Field, Counts), - io:format(FH, ", ~w", [Val]) + io:format(FH, ",~w", [Val]) end, [#counts.clients, #counts.handshakes, @@ -69,10 +69,18 @@ update_elapsed_time(Metric, DField, write_elapsed_time_head(FH) -> io:format(FH, "type, timestamp, elapsed~n", []). +metric_field(handshake_times) -> + #state.handshake_times; +metric_field(message_latencies) -> + #state.message_latencies. + write_elapsed_time(Metric, Start, End, State) -> - FH = element(Metric, State), + FH = element(metric_field(Metric), State), Elapsed = timer:now_diff(End, Start), - io:format(FH, "\"~s\", ~w, ~w~n", + + folsom_metrics:notify({Metric, Elapsed}), + + io:format(FH, "\"~s\",~w,~w~n", [State#state.type, timestamp(End), Elapsed]). timestamp({Mega, Secs, Micro}) -> @@ -84,23 +92,23 @@ inc_count(Field, Counts) -> setelement(Field, Counts, Current + 1). reducer({Start, {ws_init, Pid}}, State) -> - State2 = update_elapsed_time(#state.handshake_times, + State2 = update_elapsed_time(handshake_times, #state.clients, Pid, Start, undefined, State), State2#state{counts=inc_count(#counts.clients, State#state.counts)}; reducer({End, {ws_onopen, Pid}}, State) -> - State2 = update_elapsed_time(#state.handshake_times, + State2 = update_elapsed_time(handshake_times, #state.clients, Pid, undefined, End, State), State2#state{counts=inc_count(#counts.handshakes, State#state.counts)}; reducer({Start, {send_message, Pid, Ref}}, State) -> - State2 = update_elapsed_time(#state.message_latencies, + State2 = update_elapsed_time(message_latencies, #state.messages, {Pid, Ref}, Start, undefined, State), State2#state{counts=inc_count(#counts.messages_sent, State#state.counts)}; reducer({End, {recv_message, Pid, Ref}}, State) -> - State2 = update_elapsed_time(#state.message_latencies, + State2 = update_elapsed_time(message_latencies, #state.messages, {Pid, Ref}, undefined, End, State), State2#state{counts=inc_count(#counts.messages_recv, State#state.counts)}; @@ -121,8 +129,12 @@ reducer(E, State) -> main([Type, LogFile]) -> code:add_paths(["ebin", + "deps/bear/ebin", + "deps/folsom/ebin", "deps/eleveldb/ebin"]), + application:start(folsom), + {ok, HSFH} = file:open(filename:join(LogFile, "handshake_times.csv"), [write]), {ok, MLFH} = file:open(filename:join(LogFile, "message_latencies.csv"), @@ -130,23 +142,38 @@ main([Type, LogFile]) -> {ok, CountsFH} = file:open(filename:join(LogFile, "counts.csv"), [write]), + {ok, HistogramFH} = file:open(filename:join(LogFile, "histogram.config"), + [write]), + write_stats_head(CountsFH), write_elapsed_time_head(HSFH), write_elapsed_time_head(MLFH), + folsom_metrics:new_histogram(handshake_times), + folsom_metrics:new_histogram(message_latencies), + State = #state{type=Type, clients=dict:new(), messages=dict:new(), handshake_times=HSFH, message_latencies=MLFH}, - io:format("Aggregating stats~n", []), State2 = wsdemo_logger:foldl(fun reducer/2, State, LogFile), + + HistogramData = [ + {handshake_times, + folsom_metrics:get_histogram_statistics(handshake_times)}, + {message_latencies, + folsom_metrics:get_histogram_statistics(message_latencies)} + ], write_stats(CountsFH, State2), + io:format(HistogramFH, "~w.", [{Type, HistogramData}]), + ok = file:close(HSFH), ok = file:close(MLFH), ok = file:close(CountsFH), + ok = file:close(HistogramFH), ok; main(_) -> usage(). diff --git a/bin/convert_all_stats.sh b/bin/convert_all_stats.sh index b790b31..b26f909 100755 --- a/bin/convert_all_stats.sh +++ b/bin/convert_all_stats.sh @@ -1,6 +1,6 @@ -for db_path in data/*; do +for db_path in "$1"/*; do server_type=`basename $db_path` echo "Converting $server_type to CSV" - ./bin/events_to_csv $server_type $db_path | tee results/$server_type.events.csv + ./bin/events_to_csv $server_type $db_path > "$db_path"/events.csv done diff --git a/bin/events_to_csv b/bin/events_to_csv index 33ff3f7..b4b32c4 100755 --- a/bin/events_to_csv +++ b/bin/events_to_csv @@ -4,14 +4,17 @@ -mode(compile). -reducer({TS, {Event, Pid}}, Type) -> - write_record(TS, Type, Pid, Pid, Event), +reducer({TS, {server, {EventKey, Data}}}, Type) -> + write_record(TS, Type, server, EventKey, Data), Type; -reducer({TS, {'EXIT'=Event, Pid, Reason}}, Type) -> - write_record(TS, Type, Pid, {Pid, Reason}, Event), +reducer({TS, {EventKey, Pid}}, Type) -> + write_record(TS, Type, Pid, EventKey, Pid), Type; -reducer({TS, {Event, Pid, Ref}}, Type) -> - write_record(TS, Type, Pid, {Pid, Ref}, Event), +reducer({TS, {'EXIT'=EventKey, Pid, Reason}}, Type) -> + write_record(TS, Type, Pid, EventKey, {Pid, Reason}), + Type; +reducer({TS, {EventKey, Pid, Ref}}, Type) -> + write_record(TS, Type, Pid, EventKey, {Pid, Ref}), Type. main([Type, LogFile]) -> @@ -34,10 +37,16 @@ timestamp({Mega, Secs, Micro}) -> write_head() -> - io:format("timestamp, type, client_id, event_id, event~n", []). + io:format("timestamp,type,client_id,event_key,event_data~n", []). -write_record(TS, Type, Pid, EventKey, Event) -> - io:format("~w, \"~s\", \"~w\", \"~w\", \"~w\"~n", - [timestamp(TS), Type, Pid, EventKey, Event]). +write_record(TS, Type, Pid, EventKey, EventData) -> + DataStr = case is_integer(EventData) of + false -> + io_lib:format("\"~w\"", [EventData]); + true -> + integer_to_list(EventData) + end, + io:format("~w,\"~s\",\"~w\",\"~w\",~s~n", + [timestamp(TS), Type, Pid, EventKey, DataStr]). diff --git a/bin/meminfo b/bin/meminfo index 96be48b..666b2bf 100755 --- a/bin/meminfo +++ b/bin/meminfo @@ -2,10 +2,10 @@ PIDS=`pgrep -f "$1"` while [ 1 ] do - NUMCON=`netstat -n | awk '/ESTABLISHED/ && $4=="10.196.187.15:8000"' | wc -l` - #MEM=`ps -o rss= -p $PIDS | python -c "import sys; print sum(int(x.strip()) for x in sys.stdin)"` + SCRIPT="/ESTABLISHED/ && \$4==\"$2\"" + NUMCON=`netstat -n | awk "$SCRIPT" | wc -l` MEM=`ps -o rss= -p $PIDS | awk '{s+=$1} END {print s}'` echo -e "`date`\t`date +%s`\t$MEM\t$NUMCON" sleep 30 -done \ No newline at end of file +done diff --git a/bin/render_html b/bin/render_html new file mode 100755 index 0000000..2e0b8cc --- /dev/null +++ b/bin/render_html @@ -0,0 +1,92 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +%%! -smp enable +-mode(compile). + +compile_template() -> + erlydtl:compile(<<" + + + + + + + + + + + + + + + + + + + + {% for row in rows %}{% for col in row %} + {% endfor %} + {% endfor %} +
ImplementationHandshake Time (mean)Handshake Time (σ)Handshake Time (median)Handshake Time (95%)Handshake Time (99%)Handshake Time (99.9%)Latency (mean)Latency (σ)Latency (median)Latency (95%)Latency (99%)Latency (99.9%)Connection Timeouts
{{ col }}
">>, histogram_tmpl). + + +histogram_columns(Histogram) -> + Mean = proplists:get_value(arithmetic_mean, Histogram), + Median = proplists:get_value(median, Histogram), + Percentile = proplists:get_value(percentile, Histogram), + StdDev = proplists:get_value(standard_deviation, Histogram), + [_, {95, P95}, {99, P99}, {999, P999}] = Percentile, + lists:map(fun(N) -> trunc(N) / 1000 end, [Mean, StdDev, Median, P95, P99, P999]). + +columns(ServerName, Histograms, ConnectionTimeouts) -> + Columns = [ + list_to_binary(ServerName), + histogram_columns(proplists:get_value(handshake_times, Histograms)), + histogram_columns(proplists:get_value(message_latencies, Histograms)), + ConnectionTimeouts], + lists:flatten(Columns). + +row_score(Row) -> + [_ServerName, + HSMean, HSStdDev, _HSMedian, _HSP95, _HSP99, _HSP999, + MLMean, MLStdDev, _MLMedian, _MLP95, _MLP99, _MLP999, + ConnectionTimeouts] = Row, + {ConnectionTimeouts, HSMean, HSStdDev, MLMean, MLStdDev}. + +connection_timeouts(Root) -> + {ok, FH} = file:open(filename:join(Root, "counts.csv"), [read]), + % toss the header + io:get_line(FH, ""), + Line = io:get_line(FH, ""), + [_, _, _, Timeouts|_Rest] = string:tokens(Line, ","), + list_to_integer(Timeouts). + +main(Roots) -> + code:add_paths([ + "ebin", + "deps/erlydtl/ebin" + ]), + + compile_template(), + + Rows = lists:map(fun(Root) -> + HistogramFN = filename:join(Root, "histogram.config"), + case file:consult(HistogramFN) of + {ok, [{ServerName, Histograms}]} -> + ConnectionTimeouts = connection_timeouts(Root), + columns(ServerName, Histograms, ConnectionTimeouts); + {error, Reason} -> error({HistogramFN, Reason}) + end + end, Roots), + + SortedRows = lists:sort(fun(X,Y) -> row_score(X) =< row_score(Y) end, Rows), + + Context = [{rows, SortedRows}], + {ok, HTML} = histogram_tmpl:render(Context), + + io:format("~s~n", [HTML]). + diff --git a/bin/run_all_tests.sh b/bin/run_all_tests.sh index 85f9fea..9201b4b 100755 --- a/bin/run_all_tests.sh +++ b/bin/run_all_tests.sh @@ -1,27 +1,3 @@ -HOSTNAME=$1 -PORT=$2 +erl -boot start_sasl -pa ebin deps/*/ebin -s wsdemo_bench $@ -eval "wsdemo_bench:run_sync(), init:stop()." -do_test() { - SERVERNAME=$1 - echo "Start $SERVERNAME then press enter" - read - echo "Warming up server" - ./runtest data/warmup 60 $HOSTNAME $PORT 1000 - - echo "Start meminfo on the server then press enter" - read - echo "Running test for $SERVERNAME" - ./runtest data/$SERVERNAME 600 $HOSTNAME $PORT 10000 -} - -do_test "erlang-cowboy" -do_test "ruby-em-websockets" -do_test "go-websockets" -do_test "java-webbit" -do_test "node-websocket" -do_test "node-ws" -do_test "perl-ev" -do_test "haskell-snap" -do_test "python-tornado" -do_test "python-ws4py" diff --git a/bin/times_to_histogram b/bin/times_to_histogram new file mode 100755 index 0000000..027b754 --- /dev/null +++ b/bin/times_to_histogram @@ -0,0 +1,80 @@ +#!/usr/bin/env escript +%% -*- erlang -*- +%%! -smp enable +-mode(compile). + +row_reduce({eof}, MetricName) -> + MetricName; +row_reduce({newline, Row}, MetricName) -> + [_Type, _TS, ElapsedStr] = Row, + Elapsed = list_to_integer(ElapsedStr), + folsom_metrics:notify({MetricName, Elapsed}), + MetricName. + +compile_csv(MetricName, CSVFile) -> + {ok, IoDevice} = file:open(CSVFile, [read]), + % pop off the header + io:get_line(IoDevice, ""), + {ok, MetricName} = ecsv:process_csv_file_with(IoDevice, + fun row_reduce/2, + MetricName), + ok. + +process_framework(Root) -> + Framework = filename:basename(Root), + + folsom_metrics:new_histogram({Framework, handshake_times}), + folsom_metrics:new_histogram({Framework, message_latencies}), + + ok = compile_csv({Framework, handshake_times}, + filename:join(Root, "handshake_times.csv")), + ok = compile_csv({Framework, message_latencies}, + filename:join(Root, "message_latencies.csv")), + + {ok, HistogramFH} = file:open(filename:join(Root, "histogram.config"), + [write]), + HistogramData = [ + {handshake_times, + folsom_metrics:get_histogram_statistics( + {Framework, handshake_times})}, + {message_latencies, + folsom_metrics:get_histogram_statistics( + {Framework, message_latencies})} + ], + io:format(HistogramFH, "~w.", [{Framework, HistogramData}]), + ok = file:close(HistogramFH), + + folsom_metrics:delete_metric({Framework, handshake_times}), + folsom_metrics:delete_metric({Framework, message_latencies}), + ok. + +main(Roots) -> + code:add_paths([ + "deps/ecsv/ebin", + "deps/bear/ebin", + "deps/folsom/ebin"]), + + application:start(folsom), + + + pmap(fun process_framework/1, Roots). + +pmap(F, L) -> + Parent = self(), + pmap_results( + lists:foldl( + fun(X, I) -> + spawn_link(fun() -> Parent ! {pmap, {I, F(X)}} end), + I + 1 + end, + 0, + L + ), + [] + ). + +pmap_results(0, Rs) -> + lists:map(fun({_, R}) -> R end, lists:keysort(1, Rs)); + +pmap_results(N, Rs) -> + pmap_results(N - 1, receive {pmap, R} -> [R|Rs] end). diff --git a/competition/Makefile b/competition/Makefile index 864df92..5ac206d 100644 --- a/competition/Makefile +++ b/competition/Makefile @@ -1,9 +1,11 @@ build: + $(MAKE) -C wsdemo-playframework-2 + erlc wsdemo.erl javac -cp webbit-0.4.7-full.jar EchoWsServer.java ghc -rtsopts --make -O2 -threaded wsdemo-snap.hs + go build -o wsdemo-go-gonet wsdemo.go clean: - rm *.class - rm *snap.hi - rm *snap.o - rm wsdemo-snap + $(MAKE) -C wsdemo-playframework-2 clean + rm *.class *snap.hi *snap.o wsdemo-snap *.beam wsdemo-go-gonet + diff --git a/competition/cljwsdemo-0.1.0-SNAPSHOT-standalone.jar b/competition/cljwsdemo-0.1.0-SNAPSHOT-standalone.jar index 4d5eb4f..69d7316 100644 Binary files a/competition/cljwsdemo-0.1.0-SNAPSHOT-standalone.jar and b/competition/cljwsdemo-0.1.0-SNAPSHOT-standalone.jar differ diff --git a/competition/compile_cowboy_branch.sh b/competition/compile_cowboy_branch.sh new file mode 100755 index 0000000..63c62e8 --- /dev/null +++ b/competition/compile_cowboy_branch.sh @@ -0,0 +1,20 @@ +REPO="$1" +BRANCH="$2" +DIR="cowboy-$BRANCH" + + +function checkout { + git clone "$REPO" "$DIR" + pushd "$DIR" + git checkout "$BRANCH" + popd +} + +function compile { + pushd "$DIR" + ../../rebar get-deps compile + popd +} + +checkout +compile diff --git a/competition/log/.gitignore b/competition/log/.gitignore new file mode 100644 index 0000000..5e7d273 --- /dev/null +++ b/competition/log/.gitignore @@ -0,0 +1,4 @@ +# Ignore everything in this directory +* +# Except this file +!.gitignore diff --git a/competition/prefork.py b/competition/prefork.py new file mode 100644 index 0000000..5a2f1c1 --- /dev/null +++ b/competition/prefork.py @@ -0,0 +1,58 @@ +from multiprocessing import cpu_count +from os import fork, getpid, kill, wait +from signal import signal, SIGINT, SIGTERM, SIGQUIT +from random import seed +from time import time +from binascii import hexlify + +def _reseed_random(): + try: + from os import urandom + s = long( hexlify( urandom(16) ), 16) + except (NotImplementedError, ImportError): + s = int(time() * 1000) ^ getpid() + seed( s ) + +def prefork( num=None, fork=fork, do_seed=True, do_wait=True, do_kill=True ): + """ Forks current process num times. + + fork -- a forking function + do_seed -- True to reseed a pseudo-random generator of a child + do_wait -- True to wait for all children to exit + do_kill -- True to pass a recieved signal (INT,TERM,QUIT) to all children + + Returns a list of child pids for master or an empty list for a child. + """ + if not num: + num = cpu_count() + pids = [] + + for n in range(num): + pid = fork() + if pid: + pids.append( pid ) + else: + if do_seed: + _reseed_random() + return [] + + if do_kill: + def _pass( sig, frame ): + for pid in pids: + #print 'kill( %s, %s )' % (pid, sig) + kill( pid, sig ) + signal( SIGINT, _pass ) + signal( SIGTERM, _pass ) + signal( SIGQUIT, _pass ) + + _pids = pids[:] + while do_wait and _pids: + try: + pid, sig = wait() + except OSError: + continue + #print '%s has died' % pid + if pid in _pids: + _pids.remove( pid ) + + return pids diff --git a/competition/server.tmpl.ini b/competition/server.tmpl.ini new file mode 100644 index 0000000..e6f0959 --- /dev/null +++ b/competition/server.tmpl.ini @@ -0,0 +1,7 @@ +[program:%(server_name)s] +command=%(command)s +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/clojure-aleph.ini b/competition/servers/clojure-aleph.ini new file mode 100644 index 0000000..c891ad1 --- /dev/null +++ b/competition/servers/clojure-aleph.ini @@ -0,0 +1,7 @@ +[program:clojure-aleph] +command=java -jar cljwsdemo-0.1.0-SNAPSHOT-standalone.jar +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/erlang-cowboy.ini b/competition/servers/erlang-cowboy.ini new file mode 100644 index 0000000..2b30d7c --- /dev/null +++ b/competition/servers/erlang-cowboy.ini @@ -0,0 +1,7 @@ +[program:erlang-cowboy] +command=erl -noshell -pa ../ebin ../deps/cowboy/ebin +K true -s wsdemo +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/go-gonet.ini b/competition/servers/go-gonet.ini new file mode 100644 index 0000000..4771e61 --- /dev/null +++ b/competition/servers/go-gonet.ini @@ -0,0 +1,7 @@ +[program:go-gonet] +command=./wsdemo-go-gonet +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/haskell-snap.ini b/competition/servers/haskell-snap.ini new file mode 100644 index 0000000..bf1ac09 --- /dev/null +++ b/competition/servers/haskell-snap.ini @@ -0,0 +1,7 @@ +[program:haskell-snap] +command=./wsdemo-snap +RTS -N -A4M -qg1 +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/java-webbit.ini b/competition/servers/java-webbit.ini new file mode 100644 index 0000000..71301fd --- /dev/null +++ b/competition/servers/java-webbit.ini @@ -0,0 +1,7 @@ +[program:java-webbit] +command=java -cp webbit-0.4.7-full.jar:./ EchoWsServer +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/node-ws-cluster.ini b/competition/servers/node-ws-cluster.ini new file mode 100644 index 0000000..ba1040c --- /dev/null +++ b/competition/servers/node-ws-cluster.ini @@ -0,0 +1,7 @@ +[program:node-ws-cluster] +command=node wsdemo-ws-cluster.js +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/node-ws.ini b/competition/servers/node-ws.ini new file mode 100644 index 0000000..6ada98b --- /dev/null +++ b/competition/servers/node-ws.ini @@ -0,0 +1,7 @@ +[program:node-ws] +command=node wsdemo-ws.js +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/perl-ev.ini b/competition/servers/perl-ev.ini new file mode 100644 index 0000000..b8808c3 --- /dev/null +++ b/competition/servers/perl-ev.ini @@ -0,0 +1,7 @@ +[program:perl-ev] +command=perl wsdemo.pl +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/pypy-tornado-1.ini b/competition/servers/pypy-tornado-1.ini new file mode 100644 index 0000000..3b884c3 --- /dev/null +++ b/competition/servers/pypy-tornado-1.ini @@ -0,0 +1,7 @@ +[program:pypy-tornado-1] +command=./pypy-1.9/bin/pypy wsdemo-tornado-1.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/pypy-tornado-N.ini b/competition/servers/pypy-tornado-N.ini new file mode 100644 index 0000000..22f6505 --- /dev/null +++ b/competition/servers/pypy-tornado-N.ini @@ -0,0 +1,7 @@ +[program:pypy-tornado-N] +command=./pypy-1.9/bin/pypy wsdemo-tornado-N.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/pypy-twisted-1.ini b/competition/servers/pypy-twisted-1.ini new file mode 100644 index 0000000..2a10dab --- /dev/null +++ b/competition/servers/pypy-twisted-1.ini @@ -0,0 +1,7 @@ +[program:pypy-twisted-1] +command=./pypy-1.9/bin/pypy wsdemo-twisted-1.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/pypy-twisted-N.ini b/competition/servers/pypy-twisted-N.ini new file mode 100644 index 0000000..5eae671 --- /dev/null +++ b/competition/servers/pypy-twisted-N.ini @@ -0,0 +1,7 @@ +[program:pypy-twisted-N] +command=./pypy-1.9/bin/pypy wsdemo-twisted-N.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/python-gevent-websocket-1.ini b/competition/servers/python-gevent-websocket-1.ini new file mode 100644 index 0000000..5421ad2 --- /dev/null +++ b/competition/servers/python-gevent-websocket-1.ini @@ -0,0 +1,7 @@ +[program:python-gevent-websocket-1] +command=python wsdemo-gevent-websocket-1.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/python-gevent-websocket-N.ini b/competition/servers/python-gevent-websocket-N.ini new file mode 100644 index 0000000..58e7d3e --- /dev/null +++ b/competition/servers/python-gevent-websocket-N.ini @@ -0,0 +1,7 @@ +[program:python-gevent-websocket-N] +command=python wsdemo-gevent-websocket-N.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/python-tornado-1.ini b/competition/servers/python-tornado-1.ini new file mode 100644 index 0000000..ca9821c --- /dev/null +++ b/competition/servers/python-tornado-1.ini @@ -0,0 +1,7 @@ +[program:python-tornado-1] +command=python wsdemo-tornado-1.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/python-tornado-N.ini b/competition/servers/python-tornado-N.ini new file mode 100644 index 0000000..cb39e93 --- /dev/null +++ b/competition/servers/python-tornado-N.ini @@ -0,0 +1,7 @@ +[program:python-tornado-N] +command=python wsdemo-tornado-N.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/python-twisted-1.ini b/competition/servers/python-twisted-1.ini new file mode 100644 index 0000000..078aefd --- /dev/null +++ b/competition/servers/python-twisted-1.ini @@ -0,0 +1,7 @@ +[program:python-twisted-1] +command=python wsdemo-twisted-1.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/python-twisted-N.ini b/competition/servers/python-twisted-N.ini new file mode 100644 index 0000000..48193f7 --- /dev/null +++ b/competition/servers/python-twisted-N.ini @@ -0,0 +1,7 @@ +[program:python-twisted-N] +command=python wsdemo-twisted-N.py +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/servers/scala-play.ini b/competition/servers/scala-play.ini new file mode 100644 index 0000000..b5608de --- /dev/null +++ b/competition/servers/scala-play.ini @@ -0,0 +1,7 @@ +[program:scala-play] +command=bash ./wsdemo-playframework-2/wsdemo-1.0-SNAPSHOT/start -Dhttp.port=8000 +autostart=false +autorestart=false +startretries=0 +stopasgroup=true +killasgroup=true diff --git a/competition/supervisord.conf b/competition/supervisord.conf new file mode 100644 index 0000000..3489fce --- /dev/null +++ b/competition/supervisord.conf @@ -0,0 +1,54 @@ +; Sample supervisor config file. + +;[unix_http_server] +;file=./supervisor.sock ; (the path to the socket file) +;chmod=0700 ; socket file mode (default 0700) +;chown=nobody:nogroup ; socket file uid:gid owner +;username=user ; (default is no username (open server)) +;password=123 ; (default is no password (open server)) + +[inet_http_server] ; inet (TCP) server disabled by default +port=0.0.0.0:9001 ; (ip_address:port specifier, *:port for all iface) +;username=user ; (default is no username (open server)) +;password=123 ; (default is no password (open server)) + +[supervisord] +logfile=./log/supervisord.log ; (main log file;default $CWD/supervisord.log) +logfile_maxbytes=50MB ; (max main logfile bytes b4 rotation;default 50MB) +logfile_backups=10 ; (num of main logfile rotation backups;default 10) +loglevel=info ; (log level;default info; others: debug,warn,trace) +pidfile=./supervisord.pid ; (supervisord pidfile;default supervisord.pid) +nodaemon=false ; (start in foreground if true;default false) +minfds=1024 ; (min. avail startup file descriptors;default 1024) +minprocs=200 ; (min. avail process descriptors;default 200) +;umask=022 ; (process file creation umask;default 022) +;user=chrism ; (default is current user, required if root) +;identifier=supervisor ; (supervisord identifier, default is 'supervisor') +;directory=/tmp ; (default is not to cd during start) +;nocleanup=true ; (don't clean up tempfiles at start;default false) +;childlogdir=/tmp ; ('AUTO' child log dir, default $TEMP) +;environment=KEY=value ; (key value pairs to add to environment) +;strip_ansi=false ; (strip ansi escape codes in logs; def. false) + +; the below section must remain in the config file for RPC +; (supervisorctl/web interface) to work, additional interfaces may be +; added by defining them in separate rpcinterface: sections +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[rpcinterface:wsdemo_monitor] +supervisor.rpcinterface_factory = wsdemo_monitor:make_rpcinterface + +[supervisorctl] +;serverurl=unix://./supervisor.sock ; use a unix:// URL for a unix socket +serverurl=http://127.0.0.1:9001 ; use an http:// url to specify an inet socket +;username=chris ; should be same as http_username if set +;password=123 ; should be same as http_password if set +;prompt=mysupervisor ; cmd line prompt (default "supervisor") +;history_file=~/.sc_history ; use readline history if available + +; The below sample program section shows all possible program subsection values, +; create one or more 'real' program: sections to be able to control them under +; supervisor. +[include] +files = ./servers/*.ini diff --git a/competition/wsdemo-clojure.sh b/competition/wsdemo-clojure.sh deleted file mode 100755 index fda835f..0000000 --- a/competition/wsdemo-clojure.sh +++ /dev/null @@ -1 +0,0 @@ -java -jar cljwsdemo-0.1.0-SNAPSHOT-standalone.jar diff --git a/competition/wsdemo-em.rb b/competition/wsdemo-em.rb deleted file mode 100644 index 47f97ce..0000000 --- a/competition/wsdemo-em.rb +++ /dev/null @@ -1,12 +0,0 @@ -require 'rubygems' -require 'eventmachine' -require 'em-websocket' - -EventMachine.run { - - EventMachine::WebSocket.start(:host => "0.0.0.0", :port => 8000) do |ws| - ws.onmessage { |msg| - ws.send msg - } - end -} diff --git a/competition/wsdemo-erl.sh b/competition/wsdemo-erl.sh deleted file mode 100755 index 77b0a78..0000000 --- a/competition/wsdemo-erl.sh +++ /dev/null @@ -1 +0,0 @@ -erl -pa ../ebin ../deps/*/ebin +K true -s wsdemo \ No newline at end of file diff --git a/competition/wsdemo-gevent-websocket-1.py b/competition/wsdemo-gevent-websocket-1.py new file mode 100644 index 0000000..e416f57 --- /dev/null +++ b/competition/wsdemo-gevent-websocket-1.py @@ -0,0 +1,23 @@ +from gevent.pywsgi import WSGIServer +from geventwebsocket.handler import WebSocketHandler + +def connection( env, start ): + ws = env["wsgi.websocket"] + send, rcv = ws.send, ws.receive + try: + while True: send( rcv() ) + except: + pass + +if __name__ == "__main__": + addr = "0.0.0.0", 8000 + server = WSGIServer(addr, connection, handler_class=WebSocketHandler, backlog=768, log=None) + + if hasattr( server, 'pre_start' ): + server.pre_start() + else: + server.init_socket() + + print "Gevent + gevent-websocket (1 worker)" + + server.serve_forever() diff --git a/competition/wsdemo-gevent-websocket-N.py b/competition/wsdemo-gevent-websocket-N.py new file mode 100644 index 0000000..409b006 --- /dev/null +++ b/competition/wsdemo-gevent-websocket-N.py @@ -0,0 +1,28 @@ +from gevent.pywsgi import WSGIServer +from geventwebsocket.handler import WebSocketHandler + +from gevent import fork +from prefork import prefork, cpu_count + +def connection( env, start ): + ws = env["wsgi.websocket"] + send, rcv = ws.send, ws.receive + try: + while True: send( rcv() ) + except: + pass + +if __name__ == "__main__": + addr = "0.0.0.0", 8000 + server = WSGIServer(addr, connection, handler_class=WebSocketHandler, backlog=768, log=None) + + if hasattr( server, 'pre_start' ): + server.pre_start() + else: + server.init_socket() + + num = cpu_count() + print "Gevent + gevent-websocket (%s workers)" % num + if prefork( num, fork=fork ): exit() # gevent needs a special fork + + server.serve_forever() diff --git a/competition/wsdemo-java.sh b/competition/wsdemo-java.sh deleted file mode 100755 index ba23ec0..0000000 --- a/competition/wsdemo-java.sh +++ /dev/null @@ -1 +0,0 @@ -java -cp webbit-0.4.7-full.jar:./ EchoWsServer diff --git a/competition/wsdemo-playframework-2/Makefile b/competition/wsdemo-playframework-2/Makefile new file mode 100644 index 0000000..514aabc --- /dev/null +++ b/competition/wsdemo-playframework-2/Makefile @@ -0,0 +1,14 @@ +build: dist/wsdemo-1.0-SNAPSHOT.zip wsdemo-1.0-SNAPSHOT + +clean: + rm -rf wsdemo-1.0-SNAPSHOT/ + rm dist/wsdemo-1.0-SNAPSHOT.zip + +wsdemo-1.0-SNAPSHOT: + unzip dist/wsdemo-1.0-SNAPSHOT.zip + +dist/wsdemo-1.0-SNAPSHOT.zip: + ../play-2.0.1/play dist + + + diff --git a/competition/wsdemo-playframework-2/app/controllers/Application.scala b/competition/wsdemo-playframework-2/app/controllers/Application.scala new file mode 100644 index 0000000..2a590d9 --- /dev/null +++ b/competition/wsdemo-playframework-2/app/controllers/Application.scala @@ -0,0 +1,19 @@ +package controllers + +import play.api._ +import play.api.mvc._ +import play.api.libs.{ Comet } +import play.api.libs.iteratee._ +import play.api.libs.concurrent._ +import play.api.Play.current + +object Application extends Controller { + def index = WebSocket.using[Either[String, Array[Byte]]] { request => + val out = Enumerator.imperative[Either[String, Array[Byte]]] { + } + val in = Iteratee.foreach[Either[String, Array[Byte]]] { s => + out.push(s) + } + (in, out) + } +} diff --git a/competition/wsdemo-playframework-2/conf/application.conf b/competition/wsdemo-playframework-2/conf/application.conf new file mode 100644 index 0000000..11c9fab --- /dev/null +++ b/competition/wsdemo-playframework-2/conf/application.conf @@ -0,0 +1,47 @@ +# This is the main configuration file for the application. +# ~~~~~ + +# Secret key +# ~~~~~ +# The secret key is used to secure cryptographics functions. +# If you deploy your application to several instances be sure to use the same key! +application.secret="qru55x]uh]3ZajjQ_8ve^dc + start(8000). + +start(Port) -> + application:start(crypto), + application:start(ranch), + application:start(cowboy), + + Dispatch = [ + {'_', [{'_', ?MODULE, []}]} + ], + + cowboy:start_http( + ?MODULE, 100, + [{port, Port}, {max_connections, infinity}], + [ + {env, [{dispatch, Dispatch}]} + ] + ). + +init({tcp, http}, _Req, _Opts) -> + {upgrade, protocol, cowboy_websocket}. + +websocket_init(_TransportName, Req, State) -> + {ok, Req, State, hibernate}. + +websocket_handle(Msg, Req, State) -> + {reply, Msg, Req, State, hibernate}. + +websocket_info(_Msg, Req, State) -> + {ok, Req, State, hibernate}. + +websocket_terminate(_Reason, _Req, _State) -> + ok. diff --git a/config/localhost.config b/config/localhost.config new file mode 100644 index 0000000..89b0db6 --- /dev/null +++ b/config/localhost.config @@ -0,0 +1,14 @@ +% -*- erlang -*- +[ +{sasl, [ + {sasl_error_logger, {file, "/var/log/wsdemo-local-demo.log"}} +]}, +{wsdemo_bench, [ + {host, "127.0.0.1"}, + {port, 8000}, + {seconds, 10}, + {clients, 100}, +% {servers, ["erlang-cowboy"]}, + {db_root, "data/local-demo"} + ]} +]. diff --git a/config/moritz-server-0-test.config b/config/moritz-server-0-test.config new file mode 100644 index 0000000..c55a2ec --- /dev/null +++ b/config/moritz-server-0-test.config @@ -0,0 +1,11 @@ +% -*- erlang -*- +[ +{wsdemo_bench, [ + {host, "192.168.1.157"}, + {port, 8000}, + {seconds, 60}, + {clients, 100}, + {db_root, "/tmp"}, + {nodes, ['server-manager@127.0.0.1']} + ]} +]. diff --git a/config/moritz-server-0.config b/config/moritz-server-0.config new file mode 100644 index 0000000..d7985f2 --- /dev/null +++ b/config/moritz-server-0.config @@ -0,0 +1,13 @@ +% -*- erlang -*- +[ +{wsdemo_bench, [ + {host, "moritz-server-0"}, + {port, 8000}, + {seconds, 10}, + {clients, 10}, + {data_root, "data/"}, + {servers, [ + "erlang-cowboy" + ]} + ]} +]. diff --git a/configure_ubuntu-12.04.sh b/configure_ubuntu-12.04.sh new file mode 100755 index 0000000..7232eea --- /dev/null +++ b/configure_ubuntu-12.04.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Note: pushd/popd only work in bash + +sudo apt-get install -y \ + curl emacs unzip python-dev python-setuptools build-essential erlang-nox erlang-dev \ + libevent-dev git golang mercurial openjdk-7-jdk ruby rubygems haskell-platform + +sudo easy_install pip ws4py gevent gevent-websocket tornado twisted txws supervisor + +# Clone master +git clone git://github.com/ericmoritz/wsdemo.git wsdemo + +# Update sysctl +sudo cp wsdemo/etc/sysctl.conf /etc/ +sudo sysctl -p + +# install the wsdemo_monitor package +sudo easy_install wsdemo/priv/wsdemo_monitor + +# install pypy +pushd wsdemo/competition + if [ ! -d ./pypy-1.9 ]; then + # TODO make this platform independant + curl http://cdn.bitbucket.org/pypy/pypy/downloads/pypy-1.9-linux64.tar.bz2 | tar xj + fi + curl http://python-distribute.org/distribute_setup.py | ./pypy-1.9/bin/pypy + ./pypy-1.9/bin/easy_install pip + ./pypy-1.9/bin/pip install tornado ws4py twisted txws +popd + +# install Node +mkdir src +pushd src + curl http://nodejs.org/dist/v0.8.0/node-v0.8.0.tar.gz | tar xz + pushd node-v0.8.0 + ./configure && make && sudo make install + popd +popd + +# install Play +pushd wsdemo/competition + if [ ! -d ./play-2.0.1 ]; then + curl -O http://download.playframework.org/releases/play-2.0.1.zip + unzip play-2.0.1.zip + fi +popd + +npm install websocket ws +sudo go get code.google.com/p/go.net/websocket +sudo gem install em-websocket + +sudo cabal update +sudo cabal install snap-server snap-core websockets websockets-snap + +sudo apt-get install -y libadns1-dev +echo -e "y\ny\no conf prerequisites_policy follow\no conf commit" | sudo cpan +sudo cpan Protocol::WebSocket +sudo cpan YAML +sudo cpan EV +sudo cpan EV::ADNS +yes | sudo cpan IO::Stream diff --git a/configure_ubuntu.sh b/configure_ubuntu.sh deleted file mode 100755 index 0b407c0..0000000 --- a/configure_ubuntu.sh +++ /dev/null @@ -1,30 +0,0 @@ -sudo apt-get install -y emacs python-dev python-setuptools build-essential erlang-nox libevent-dev git golang mercurial default-jdk ruby rubygems haskell-platform -sudo easy_install ws4py gevent - -git clone git://github.com/ericmoritz/wsdemo.git wsdemo - -sudo cp wsdemo/etc/sysctl.conf /etc/ -sudo sysctl -p - -mkdir src -pushd src - curl http://nodejs.org/dist/v0.7.11/node-v0.7.11.tar.gz | tar xz - pushd node-v0.7.11 - ./configure && make && sudo make install - popd -popd - -npm install websocket -npm install ws -sudo go get code.google.com/p/go.net/websocket -sudo gem install em-websocket - -sudo cabal update -sudo cabal install snap-server snap-core websockets websockets-snap - -sudo apt-get install -y libadns1-dev -echo -e "y\ny\no conf prerequisites_policy follow\no conf commit" | sudo cpan -sudo cpan Protocol::WebSocket -sudo cpan EV::ADNS -yes | sudo cpan IO::Stream - diff --git a/etc/sysctl.conf b/etc/sysctl.conf index 44abe05..ef6b2cf 100644 --- a/etc/sysctl.conf +++ b/etc/sysctl.conf @@ -1,4 +1,5 @@ # General gigabit tuning: +net.core.somaxconn = 16384 net.core.rmem_max = 16777216 net.core.wmem_max = 16777216 net.ipv4.tcp_rmem = 4096 87380 16777216 diff --git a/priv/summerize.r b/priv/summerize.r deleted file mode 100755 index ff59840..0000000 --- a/priv/summerize.r +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env Rscript --vanilla - -stats <- function(times) { - elapsed = times$elapsed / 1000 - quant = quantile(elapsed, c(.75, .95, .99, .999)) - s <- c(mean(elapsed), - min(elapsed), - median(elapsed), - quant, - max(elapsed)) - names(s) <- c("Mean", "Min", "Median", names(quant), "Max") - return(s) -} - -root <- commandArgs(trailingOnly=T)[1] -handshake_times_csv = file.path(root, "handshake_times.csv") -latencies_csv = file.path(root, "message_latencies.csv") - -handshakes <- read.csv(handshake_times_csv, head=T, sep=",", as.is=T, strip.white=T) -latencies <- read.csv(latencies_csv, head=T, sep=",", as.is=T, strip.white=T) - -print(root) -print("Handshake Times") -print(stats(handshakes)) -print("Latencies") -print(stats(latencies)) diff --git a/priv/wsdemo_monitor/build/lib/wsdemo_monitor/__init__.py b/priv/wsdemo_monitor/build/lib/wsdemo_monitor/__init__.py new file mode 100644 index 0000000..c81a24c --- /dev/null +++ b/priv/wsdemo_monitor/build/lib/wsdemo_monitor/__init__.py @@ -0,0 +1,47 @@ +import re +import subprocess +from supervisor import rpcinterface +from supervisor.xmlrpc import Faults +from supervisor.xmlrpc import RPCError + + +# OSX's netstat ips look like 127.0.0.0.8000 (really?) +OSX_IP_PAT = re.compile(r"(\d+\.\d+\.\d+\.\d+)\.(\d+)") +def fix_osx_ips(ip): + match = OSX_IP_PAT.match(ip) + if match: + return "%s:%s" % (match.groups()) + else: + return ip + + +def get_connections(hostname): + lines = subprocess.check_output(["netstat", "-n"]).splitlines() + rows = (line.split() for line in lines if "ESTABLISHED" in line) + rows = (row for row in rows if fix_osx_ips(row[4]) == hostname) + return len(list(rows)) + + +def get_rss(pid): + return sum(map(int,subprocess.check_output(["ps", "-o rss=", "-g", str(pid)]).split())) + + +class WSDemoMonitor(rpcinterface.SupervisorNamespaceRPCInterface): + def __init__(self, supervisord, **config): + rpcinterface.SupervisorNamespaceRPCInterface.__init__(self, supervisord) + + def memusage(self, name): + info = self.getProcessInfo(name) + + if info['pid'] == 0: + raise RPCError(Faults.NOT_RUNNING) + + return get_rss(info['pid']) + + def connections(self, hostname): + return get_connections(hostname) + + +def make_rpcinterface(supervisord, **config): + return WSDemoMonitor(supervisord, **config) + diff --git a/priv/wsdemo_monitor/setup.cfg b/priv/wsdemo_monitor/setup.cfg new file mode 100644 index 0000000..8b0b6ee --- /dev/null +++ b/priv/wsdemo_monitor/setup.cfg @@ -0,0 +1,6 @@ +[egg_info] +tag_build = dev +tag_svn_revision = true + +[easy_install] + diff --git a/priv/wsdemo_monitor/setup.py b/priv/wsdemo_monitor/setup.py new file mode 100644 index 0000000..f68ad52 --- /dev/null +++ b/priv/wsdemo_monitor/setup.py @@ -0,0 +1,26 @@ +from setuptools import setup, find_packages +import sys, os + +version = '0.1' + +setup(name='wsdemo_monitor', + version=version, + description="", + long_description="""\ +""", + classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers + keywords='', + author='', + author_email='', + url='', + license='', + packages=find_packages(exclude=['ez_setup', 'examples', 'tests']), + include_package_data=True, + zip_safe=False, + install_requires=[ + # -*- Extra requirements: -*- + ], + entry_points=""" + # -*- Entry points: -*- + """, + ) diff --git a/priv/wsdemo_monitor/wsdemo_monitor.egg-info/PKG-INFO b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/PKG-INFO new file mode 100644 index 0000000..c10cbfe --- /dev/null +++ b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/PKG-INFO @@ -0,0 +1,10 @@ +Metadata-Version: 1.0 +Name: wsdemo-monitor +Version: 0.1dev +Summary: UNKNOWN +Home-page: UNKNOWN +Author: UNKNOWN +Author-email: UNKNOWN +License: UNKNOWN +Description: UNKNOWN +Platform: UNKNOWN diff --git a/priv/wsdemo_monitor/wsdemo_monitor.egg-info/SOURCES.txt b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/SOURCES.txt new file mode 100644 index 0000000..80035ee --- /dev/null +++ b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/SOURCES.txt @@ -0,0 +1,9 @@ +setup.cfg +setup.py +wsdemo_monitor/__init__.py +wsdemo_monitor.egg-info/PKG-INFO +wsdemo_monitor.egg-info/SOURCES.txt +wsdemo_monitor.egg-info/dependency_links.txt +wsdemo_monitor.egg-info/entry_points.txt +wsdemo_monitor.egg-info/not-zip-safe +wsdemo_monitor.egg-info/top_level.txt \ No newline at end of file diff --git a/priv/wsdemo_monitor/wsdemo_monitor.egg-info/dependency_links.txt b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/priv/wsdemo_monitor/wsdemo_monitor.egg-info/entry_points.txt b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/entry_points.txt new file mode 100644 index 0000000..5d3e5f6 --- /dev/null +++ b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/entry_points.txt @@ -0,0 +1,3 @@ + + # -*- Entry points: -*- + \ No newline at end of file diff --git a/priv/wsdemo_monitor/wsdemo_monitor.egg-info/not-zip-safe b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/not-zip-safe new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/not-zip-safe @@ -0,0 +1 @@ + diff --git a/priv/wsdemo_monitor/wsdemo_monitor.egg-info/top_level.txt b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/top_level.txt new file mode 100644 index 0000000..6334bf9 --- /dev/null +++ b/priv/wsdemo_monitor/wsdemo_monitor.egg-info/top_level.txt @@ -0,0 +1 @@ +wsdemo_monitor diff --git a/priv/wsdemo_monitor/wsdemo_monitor/__init__.py b/priv/wsdemo_monitor/wsdemo_monitor/__init__.py new file mode 100644 index 0000000..6733a5a --- /dev/null +++ b/priv/wsdemo_monitor/wsdemo_monitor/__init__.py @@ -0,0 +1,47 @@ +import re +import subprocess +from supervisor import rpcinterface +from supervisor.xmlrpc import Faults +from supervisor.xmlrpc import RPCError + + +# OSX's netstat ips look like 127.0.0.0.8000 (really?) +OSX_IP_PAT = re.compile(r"(\d+\.\d+\.\d+\.\d+)\.(\d+)") +def fix_osx_ips(ip): + match = OSX_IP_PAT.match(ip) + if match: + return "%s:%s" % (match.groups()) + else: + return ip + + +def get_connections(hostname): + lines = subprocess.check_output(["netstat", "-n"]).splitlines() + rows = (line.split() for line in lines if "ESTABLISHED" in line) + rows = (row for row in rows if fix_osx_ips(row[3]) == hostname) + return len(list(rows)) + + +def get_rss(pid): + return sum(map(int,subprocess.check_output(["ps", "-o", "rss=", "-g", str(pid), "-p", str(pid)]).split())) + + +class WSDemoMonitor(rpcinterface.SupervisorNamespaceRPCInterface): + def __init__(self, supervisord, **config): + rpcinterface.SupervisorNamespaceRPCInterface.__init__(self, supervisord) + + def memusage(self, name): + info = self.getProcessInfo(name) + + if info['pid'] == 0: + raise RPCError(Faults.NOT_RUNNING) + + return get_rss(info['pid']) + + def connections(self, hostname): + return get_connections(hostname) + + +def make_rpcinterface(supervisord, **config): + return WSDemoMonitor(supervisord, **config) + diff --git a/rebar.config b/rebar.config index eb9ff01..f476a8f 100644 --- a/rebar.config +++ b/rebar.config @@ -1,4 +1,7 @@ {deps, [ + {ecsv, ".*", {git, "https://github.com/refuge/ecsv.git", "master"}}, + {erlydtl, ".*", {git, "git://github.com/evanmiller/erlydtl.git", "master"}}, + {xmlrpc, ".*", {git, "git://github.com/etnt/xmlrpc.git", "master"}}, {eleveldb, ".*", {git, "git://github.com/basho/eleveldb.git", {tag, "1.1.0"}}}, {cowboy, ".*", {git, "https://github.com/extend/cowboy.git", {tag, "0.6.0"}}}, {folsom, ".*", {git, "git://github.com/boundary/folsom.git", "master"}}, diff --git a/results.csv b/results.csv deleted file mode 100644 index 238156b..0000000 --- a/results.csv +++ /dev/null @@ -1,7 +0,0 @@ -implementation,connection.time,latency,messages,connections,timeouts -Erlang,865,17,2849294,10000,0 -Haskell (Snap),168,227,1187413,4996,108 -Java (Webbit),567,835,1028390,4637,157 -Go,284,18503,2398180,9775,225 -Node.js (websocket),768,42580,1170847,5701,4299 -Python (ws4py),1561,34889,1052996,4792,5208 diff --git a/results.md b/results.md deleted file mode 100644 index 2727f61..0000000 --- a/results.md +++ /dev/null @@ -1,108 +0,0 @@ -# C10K Websocket Test -*NOTE* I am planning on rerunning these benchmarks with the new stats collecting code using 64bit m1.large instances. -This document has been truncated until the new benchmarks can be reran. The [previous results](https://github.com/ericmoritz/wsdemo/blob/results-v1/results.md) are still available - -## Methodology - -This benchmark starts a new client every 1ms, each client sends a -timestamp message to the server every second and the server echos that -message back to the client. - - -I will start two 64 bit m1.large AWS instances running Ubuntu 12.04. -The following command will be ran on both base systems: - - curl https://github.com/ericmoritz/wsdemo/blob/master/configure_ubuntu.sh | bash - -Both EC2 instances are configured with the following `etc/sysctl.conf` -file: - - # General gigabit tuning: - net.core.rmem_max = 16777216 - net.core.wmem_max = 16777216 - net.ipv4.tcp_rmem = 4096 87380 16777216 - net.ipv4.tcp_wmem = 4096 65536 16777216 - net.ipv4.tcp_syncookies = 1 - # this gives the kernel more memory for tcp - # which you need with many (100k+) open socket connections - net.ipv4.tcp_mem = 50576 64768 98152 - net.core.netdev_max_backlog = 2500 - -On one instance I will start each of the servers as follows: - - sudo bash - ulimit -n 999999 - $SERVER_CMD - -and the other instance I will run the `./runtest` command as: - - sudo bash - ulimit -n 999999 - ./runtest results/${SERVER_TYPE}.dat 300 $SERVER_HOSTNAME 8000 10000 - -The clients communicate to the server via the server's public IP. - - -## Statistic data - -The data generated is an length encoded (2 bytes) Binary Erlang term. The term is in the following formats - - {timestamp(), {ws_init, pid()}} - {timestamp(), {ws_onopen, pid()}} - {timestamp(), {message_send, pid(), ref()}} - {timestamp(), {message_recv, pid(), ref()}} - {timestamp(), {'EXIT', pid(), connection_timeout}} - {timestamp(), {'EXIT', pid(), normal}} - {timestamp(), {'EXIT', pid(), Reason :: any()}} - -To dump the messages to STDOUT run the following command: - - ./cat_stats $FILENAME - -To generate aggregate statistics: - - ./compile_stats $FILENAME - -## Aggregate Statistic Definitions - -### clients - -Total number of clients started - -### handshakes - -Total number of clients that successfully completed the Websocket handshake - -### handshake_time - -Elapsed time between starting the client and when the websocket handshake finished. - -### messages_sent - -Total number of messages sent by the client - -### messages_recv - -Total number of messages received by the client - -### message_latency - -Elapsed time for a message sent from the client to echo -back to the client by the server. - -### connection_timeouts - -Total number of connection timeouts. The timeout is currently hard -coded as 2 seconds. This means that any TCP connection that takes -longer than 2 seconds to be accepted, this number will be incremented -and the client crashes - -### crashes - -Number of crashed clients from a reason other than timeout - -## Summary - -*TODO* The statistic gathering code has been refactored and the tests need to be rerun. The -[previous results](https://github.com/ericmoritz/wsdemo/blob/results-v1/results.md) are still available. - diff --git a/results/erlang-cowboy.meminfo b/results/erlang-cowboy.meminfo deleted file mode 100644 index 0ea6e2e..0000000 --- a/results/erlang-cowboy.meminfo +++ /dev/null @@ -1,21 +0,0 @@ -Sat Jun 16 09:28:22 UTC 2012 1339838902 527148 0 -Sat Jun 16 09:28:53 UTC 2012 1339838933 570820 10000 -Sat Jun 16 09:29:23 UTC 2012 1339838963 571084 10000 -Sat Jun 16 09:29:53 UTC 2012 1339838993 571084 10000 -Sat Jun 16 09:30:23 UTC 2012 1339839023 571084 10000 -Sat Jun 16 09:30:54 UTC 2012 1339839054 571084 10000 -Sat Jun 16 09:31:24 UTC 2012 1339839084 571084 10000 -Sat Jun 16 09:31:54 UTC 2012 1339839114 571084 10000 -Sat Jun 16 09:32:24 UTC 2012 1339839144 569468 10000 -Sat Jun 16 09:32:55 UTC 2012 1339839175 569468 10000 -Sat Jun 16 09:33:25 UTC 2012 1339839205 569600 10000 -Sat Jun 16 09:33:55 UTC 2012 1339839235 569556 10000 -Sat Jun 16 09:34:25 UTC 2012 1339839265 570348 10000 -Sat Jun 16 09:34:56 UTC 2012 1339839296 570348 10000 -Sat Jun 16 09:35:26 UTC 2012 1339839326 570348 10000 -Sat Jun 16 09:35:56 UTC 2012 1339839356 569644 10000 -Sat Jun 16 09:36:26 UTC 2012 1339839386 569644 10000 -Sat Jun 16 09:36:57 UTC 2012 1339839417 569640 10000 -Sat Jun 16 09:37:27 UTC 2012 1339839447 569640 10000 -Sat Jun 16 09:37:57 UTC 2012 1339839477 569640 10000 -Sat Jun 16 09:38:27 UTC 2012 1339839507 569484 26 diff --git a/results/go-websockets.meminfo b/results/go-websockets.meminfo deleted file mode 100644 index ea90293..0000000 --- a/results/go-websockets.meminfo +++ /dev/null @@ -1,21 +0,0 @@ -Sat Jun 16 09:50:31 UTC 2012 1339840231 38904 0 -Sat Jun 16 09:51:01 UTC 2012 1339840261 335176 10000 -Sat Jun 16 09:51:31 UTC 2012 1339840291 404436 10000 -Sat Jun 16 09:52:01 UTC 2012 1339840321 440720 10000 -Sat Jun 16 09:52:32 UTC 2012 1339840352 554068 10000 -Sat Jun 16 09:53:02 UTC 2012 1339840382 666956 10000 -Sat Jun 16 09:53:32 UTC 2012 1339840412 779496 10000 -Sat Jun 16 09:54:02 UTC 2012 1339840442 791196 10000 -Sat Jun 16 09:54:33 UTC 2012 1339840473 792508 10000 -Sat Jun 16 09:55:03 UTC 2012 1339840503 793740 10000 -Sat Jun 16 09:55:33 UTC 2012 1339840533 794268 10000 -Sat Jun 16 09:56:03 UTC 2012 1339840563 839380 10000 -Sat Jun 16 09:56:34 UTC 2012 1339840594 846616 10000 -Sat Jun 16 09:57:04 UTC 2012 1339840624 847472 10000 -Sat Jun 16 09:57:34 UTC 2012 1339840654 847732 10000 -Sat Jun 16 09:58:05 UTC 2012 1339840685 848520 10000 -Sat Jun 16 09:58:35 UTC 2012 1339840715 849048 10000 -Sat Jun 16 09:59:05 UTC 2012 1339840745 849048 10000 -Sat Jun 16 09:59:35 UTC 2012 1339840775 849312 10000 -Sat Jun 16 10:00:06 UTC 2012 1339840806 851148 10000 -Sat Jun 16 10:00:36 UTC 2012 1339840836 852192 0 diff --git a/results/haskell-snap.meminfo b/results/haskell-snap.meminfo deleted file mode 100644 index 511f089..0000000 --- a/results/haskell-snap.meminfo +++ /dev/null @@ -1,21 +0,0 @@ -Sat Jun 16 13:08:14 UTC 2012 1339852094 25748 0 -Sat Jun 16 13:08:44 UTC 2012 1339852124 852028 9759 -Sat Jun 16 13:09:14 UTC 2012 1339852154 1045016 9759 -Sat Jun 16 13:09:44 UTC 2012 1339852184 1118936 9759 -Sat Jun 16 13:10:15 UTC 2012 1339852215 1132556 9759 -Sat Jun 16 13:10:45 UTC 2012 1339852245 1167140 9759 -Sat Jun 16 13:11:15 UTC 2012 1339852275 1168196 9759 -Sat Jun 16 13:11:46 UTC 2012 1339852306 1168460 9759 -Sat Jun 16 13:12:16 UTC 2012 1339852336 1169732 9759 -Sat Jun 16 13:12:46 UTC 2012 1339852366 1169732 9759 -Sat Jun 16 13:13:16 UTC 2012 1339852396 1169732 9759 -Sat Jun 16 13:13:47 UTC 2012 1339852427 1171316 9759 -Sat Jun 16 13:14:17 UTC 2012 1339852457 1171580 9759 -Sat Jun 16 13:14:47 UTC 2012 1339852487 1191644 9759 -Sat Jun 16 13:15:18 UTC 2012 1339852518 1191644 9759 -Sat Jun 16 13:15:48 UTC 2012 1339852548 1191644 9759 -Sat Jun 16 13:16:18 UTC 2012 1339852578 1191644 9759 -Sat Jun 16 13:16:48 UTC 2012 1339852608 1191644 9759 -Sat Jun 16 13:17:19 UTC 2012 1339852639 1192700 9759 -Sat Jun 16 13:17:49 UTC 2012 1339852669 1192700 9759 -Sat Jun 16 13:18:19 UTC 2012 1339852699 739116 212 diff --git a/results/java-webbit.meminfo b/results/java-webbit.meminfo deleted file mode 100644 index 87dd3c2..0000000 --- a/results/java-webbit.meminfo +++ /dev/null @@ -1,26 +0,0 @@ -Sat Jun 16 10:03:51 UTC 2012 1339841031 74300 0 -Sat Jun 16 10:04:22 UTC 2012 1339841062 218276 9940 -Sat Jun 16 10:04:52 UTC 2012 1339841092 331748 9940 -Sat Jun 16 10:05:22 UTC 2012 1339841122 333352 9940 -Sat Jun 16 10:05:52 UTC 2012 1339841152 334344 9940 -Sat Jun 16 10:06:23 UTC 2012 1339841183 334388 9940 -Sat Jun 16 10:06:53 UTC 2012 1339841213 334160 9940 -Sat Jun 16 10:07:23 UTC 2012 1339841243 334376 9940 -Sat Jun 16 10:07:53 UTC 2012 1339841273 334084 9940 -Sat Jun 16 10:08:24 UTC 2012 1339841304 333264 9940 -Sat Jun 16 10:08:54 UTC 2012 1339841334 328092 9940 -Sat Jun 16 10:09:24 UTC 2012 1339841364 322728 9940 -Sat Jun 16 10:09:54 UTC 2012 1339841394 321020 9940 -Sat Jun 16 10:10:25 UTC 2012 1339841425 319592 9940 -Sat Jun 16 10:10:55 UTC 2012 1339841455 319376 9940 -Sat Jun 16 10:11:25 UTC 2012 1339841485 318832 9940 -Sat Jun 16 10:11:55 UTC 2012 1339841515 318360 9940 -Sat Jun 16 10:12:26 UTC 2012 1339841546 317832 9940 -Sat Jun 16 10:12:56 UTC 2012 1339841576 316916 9940 -Sat Jun 16 10:13:26 UTC 2012 1339841606 316564 9940 -Sat Jun 16 10:13:56 UTC 2012 1339841636 317528 106 -Sat Jun 16 10:14:26 UTC 2012 1339841666 317528 106 -Sat Jun 16 10:14:56 UTC 2012 1339841696 317528 106 -Sat Jun 16 10:15:26 UTC 2012 1339841726 317528 106 -Sat Jun 16 10:15:56 UTC 2012 1339841756 317528 106 -Sat Jun 16 10:16:26 UTC 2012 1339841786 317528 106 diff --git a/results/node-websocket.meminfo b/results/node-websocket.meminfo deleted file mode 100644 index 62a93f3..0000000 --- a/results/node-websocket.meminfo +++ /dev/null @@ -1,22 +0,0 @@ -Sat Jun 16 10:18:55 UTC 2012 1339841935 67256 0 -Sat Jun 16 10:19:25 UTC 2012 1339841965 185776 7529 -Sat Jun 16 10:19:55 UTC 2012 1339841995 187916 7529 -Sat Jun 16 10:20:26 UTC 2012 1339842026 189388 7529 -Sat Jun 16 10:20:56 UTC 2012 1339842056 189388 7529 -Sat Jun 16 10:21:26 UTC 2012 1339842086 189652 7529 -Sat Jun 16 10:21:56 UTC 2012 1339842116 189992 7529 -Sat Jun 16 10:22:27 UTC 2012 1339842147 189992 7529 -Sat Jun 16 10:22:57 UTC 2012 1339842177 190196 7529 -Sat Jun 16 10:23:27 UTC 2012 1339842207 190196 7529 -Sat Jun 16 10:23:57 UTC 2012 1339842237 190196 7529 -Sat Jun 16 10:24:27 UTC 2012 1339842267 190196 7529 -Sat Jun 16 10:24:58 UTC 2012 1339842298 190196 7529 -Sat Jun 16 10:25:28 UTC 2012 1339842328 190196 7529 -Sat Jun 16 10:25:58 UTC 2012 1339842358 190196 7529 -Sat Jun 16 10:26:28 UTC 2012 1339842388 190196 7529 -Sat Jun 16 10:26:58 UTC 2012 1339842418 191160 7529 -Sat Jun 16 10:27:29 UTC 2012 1339842449 191160 7529 -Sat Jun 16 10:27:59 UTC 2012 1339842479 191540 7529 -Sat Jun 16 10:28:29 UTC 2012 1339842509 190164 7529 -Sat Jun 16 10:28:59 UTC 2012 1339842539 192012 446 -Sat Jun 16 10:29:29 UTC 2012 1339842569 35476 0 diff --git a/results/node-ws.meminfo b/results/node-ws.meminfo deleted file mode 100644 index 89d4e17..0000000 --- a/results/node-ws.meminfo +++ /dev/null @@ -1,24 +0,0 @@ -Sat Jun 16 11:38:22 UTC 2012 1339846702 35024 0 -Sat Jun 16 11:38:53 UTC 2012 1339846733 181612 7365 -Sat Jun 16 11:39:23 UTC 2012 1339846763 190096 7375 -Sat Jun 16 11:39:53 UTC 2012 1339846793 190284 7375 -Sat Jun 16 11:40:23 UTC 2012 1339846823 190284 7375 -Sat Jun 16 11:40:53 UTC 2012 1339846853 189492 7375 -Sat Jun 16 11:41:24 UTC 2012 1339846884 192768 7375 -Sat Jun 16 11:41:54 UTC 2012 1339846914 191976 7375 -Sat Jun 16 11:42:24 UTC 2012 1339846944 191976 7375 -Sat Jun 16 11:42:55 UTC 2012 1339846975 192844 7375 -Sat Jun 16 11:43:25 UTC 2012 1339847005 191208 7375 -Sat Jun 16 11:43:55 UTC 2012 1339847035 192612 7375 -Sat Jun 16 11:44:25 UTC 2012 1339847065 192260 7375 -Sat Jun 16 11:44:55 UTC 2012 1339847095 192788 7375 -Sat Jun 16 11:45:26 UTC 2012 1339847126 191208 7375 -Sat Jun 16 11:45:56 UTC 2012 1339847156 192788 7375 -Sat Jun 16 11:46:26 UTC 2012 1339847186 191996 7375 -Sat Jun 16 11:46:56 UTC 2012 1339847216 193332 7375 -Sat Jun 16 11:47:26 UTC 2012 1339847246 192428 7375 -Sat Jun 16 11:47:57 UTC 2012 1339847277 191820 7375 -Sat Jun 16 11:48:27 UTC 2012 1339847307 192524 592 -Sat Jun 16 11:48:57 UTC 2012 1339847337 67196 592 -Sat Jun 16 11:49:27 UTC 2012 1339847367 67196 592 -Sat Jun 16 11:49:57 UTC 2012 1339847397 67196 592 diff --git a/results/perl-ev.meminfo b/results/perl-ev.meminfo deleted file mode 100644 index 655cdc9..0000000 --- a/results/perl-ev.meminfo +++ /dev/null @@ -1,142 +0,0 @@ -Sat Jun 16 11:52:25 UTC 2012 1339847545 19772 0 -Sat Jun 16 11:52:55 UTC 2012 1339847575 74220 4825 -Sat Jun 16 11:53:25 UTC 2012 1339847605 75540 4867 -Sat Jun 16 11:53:55 UTC 2012 1339847635 75540 4867 -Sat Jun 16 11:54:26 UTC 2012 1339847666 75540 4867 -Sat Jun 16 11:54:56 UTC 2012 1339847696 75540 4867 -Sat Jun 16 11:55:26 UTC 2012 1339847726 75540 4867 -Sat Jun 16 11:55:56 UTC 2012 1339847756 75540 4867 -Sat Jun 16 11:56:26 UTC 2012 1339847786 75540 4867 -Sat Jun 16 11:56:56 UTC 2012 1339847816 75540 4867 -Sat Jun 16 11:57:26 UTC 2012 1339847846 75540 4867 -Sat Jun 16 11:57:57 UTC 2012 1339847877 75540 4867 -Sat Jun 16 11:58:27 UTC 2012 1339847907 75540 4867 -Sat Jun 16 11:58:57 UTC 2012 1339847937 75540 4867 -Sat Jun 16 11:59:27 UTC 2012 1339847967 75540 4867 -Sat Jun 16 11:59:57 UTC 2012 1339847997 75540 4867 -Sat Jun 16 12:00:27 UTC 2012 1339848027 75540 4867 -Sat Jun 16 12:00:57 UTC 2012 1339848057 75540 4867 -Sat Jun 16 12:01:27 UTC 2012 1339848087 75540 4867 -Sat Jun 16 12:01:58 UTC 2012 1339848118 75540 4867 -Sat Jun 16 12:02:28 UTC 2012 1339848148 75540 4867 -Sat Jun 16 12:02:58 UTC 2012 1339848178 75864 28 -Sat Jun 16 12:03:28 UTC 2012 1339848208 75864 28 -Sat Jun 16 12:03:58 UTC 2012 1339848238 75864 28 -Sat Jun 16 12:04:28 UTC 2012 1339848268 75864 28 -Sat Jun 16 12:04:58 UTC 2012 1339848298 75864 28 -Sat Jun 16 12:05:28 UTC 2012 1339848328 75864 28 -Sat Jun 16 12:05:58 UTC 2012 1339848358 75864 28 -Sat Jun 16 12:06:28 UTC 2012 1339848388 75864 28 -Sat Jun 16 12:06:58 UTC 2012 1339848418 75864 28 -Sat Jun 16 12:07:28 UTC 2012 1339848448 75864 28 -Sat Jun 16 12:07:58 UTC 2012 1339848478 75864 28 -Sat Jun 16 12:08:28 UTC 2012 1339848508 75864 28 -Sat Jun 16 12:08:58 UTC 2012 1339848538 75864 28 -Sat Jun 16 12:09:28 UTC 2012 1339848568 75864 28 -Sat Jun 16 12:09:58 UTC 2012 1339848598 75864 28 -Sat Jun 16 12:10:29 UTC 2012 1339848629 75864 28 -Sat Jun 16 12:10:59 UTC 2012 1339848659 75864 28 -Sat Jun 16 12:11:29 UTC 2012 1339848689 75864 28 -Sat Jun 16 12:11:59 UTC 2012 1339848719 75864 28 -Sat Jun 16 12:12:29 UTC 2012 1339848749 75864 28 -Sat Jun 16 12:12:59 UTC 2012 1339848779 75864 28 -Sat Jun 16 12:13:29 UTC 2012 1339848809 75864 28 -Sat Jun 16 12:13:59 UTC 2012 1339848839 75864 28 -Sat Jun 16 12:14:29 UTC 2012 1339848869 75864 28 -Sat Jun 16 12:14:59 UTC 2012 1339848899 75864 28 -Sat Jun 16 12:15:29 UTC 2012 1339848929 75864 28 -Sat Jun 16 12:15:59 UTC 2012 1339848959 75864 28 -Sat Jun 16 12:16:29 UTC 2012 1339848989 75864 28 -Sat Jun 16 12:16:59 UTC 2012 1339849019 75864 28 -Sat Jun 16 12:17:29 UTC 2012 1339849049 75864 28 -Sat Jun 16 12:17:59 UTC 2012 1339849079 75864 28 -Sat Jun 16 12:18:29 UTC 2012 1339849109 75864 28 -Sat Jun 16 12:18:59 UTC 2012 1339849139 75864 28 -Sat Jun 16 12:19:29 UTC 2012 1339849169 75864 28 -Sat Jun 16 12:19:59 UTC 2012 1339849199 75864 28 -Sat Jun 16 12:20:29 UTC 2012 1339849229 75864 28 -Sat Jun 16 12:21:00 UTC 2012 1339849260 75864 28 -Sat Jun 16 12:21:30 UTC 2012 1339849290 75864 28 -Sat Jun 16 12:22:00 UTC 2012 1339849320 75864 28 -Sat Jun 16 12:22:30 UTC 2012 1339849350 75864 28 -Sat Jun 16 12:23:00 UTC 2012 1339849380 75864 28 -Sat Jun 16 12:23:30 UTC 2012 1339849410 75864 28 -Sat Jun 16 12:24:00 UTC 2012 1339849440 75864 28 -Sat Jun 16 12:24:30 UTC 2012 1339849470 75864 28 -Sat Jun 16 12:25:00 UTC 2012 1339849500 75864 28 -Sat Jun 16 12:25:30 UTC 2012 1339849530 75864 28 -Sat Jun 16 12:26:00 UTC 2012 1339849560 75864 28 -Sat Jun 16 12:26:30 UTC 2012 1339849590 75864 28 -Sat Jun 16 12:27:00 UTC 2012 1339849620 75864 28 -Sat Jun 16 12:27:30 UTC 2012 1339849650 75864 28 -Sat Jun 16 12:28:00 UTC 2012 1339849680 75864 28 -Sat Jun 16 12:28:30 UTC 2012 1339849710 75864 28 -Sat Jun 16 12:29:00 UTC 2012 1339849740 75864 28 -Sat Jun 16 12:29:30 UTC 2012 1339849770 75864 28 -Sat Jun 16 12:30:00 UTC 2012 1339849800 75864 28 -Sat Jun 16 12:30:30 UTC 2012 1339849830 75864 28 -Sat Jun 16 12:31:00 UTC 2012 1339849860 75864 28 -Sat Jun 16 12:31:30 UTC 2012 1339849890 75864 28 -Sat Jun 16 12:32:01 UTC 2012 1339849921 75864 28 -Sat Jun 16 12:32:31 UTC 2012 1339849951 75864 28 -Sat Jun 16 12:33:01 UTC 2012 1339849981 75864 28 -Sat Jun 16 12:33:31 UTC 2012 1339850011 75864 28 -Sat Jun 16 12:34:01 UTC 2012 1339850041 75864 28 -Sat Jun 16 12:34:31 UTC 2012 1339850071 75864 28 -Sat Jun 16 12:35:01 UTC 2012 1339850101 75864 28 -Sat Jun 16 12:35:31 UTC 2012 1339850131 75864 28 -Sat Jun 16 12:36:01 UTC 2012 1339850161 75864 28 -Sat Jun 16 12:36:31 UTC 2012 1339850191 75864 28 -Sat Jun 16 12:37:01 UTC 2012 1339850221 75864 28 -Sat Jun 16 12:37:31 UTC 2012 1339850251 75864 28 -Sat Jun 16 12:38:01 UTC 2012 1339850281 75864 28 -Sat Jun 16 12:38:31 UTC 2012 1339850311 75864 28 -Sat Jun 16 12:39:01 UTC 2012 1339850341 75864 28 -Sat Jun 16 12:39:31 UTC 2012 1339850371 75864 28 -Sat Jun 16 12:40:01 UTC 2012 1339850401 75864 28 -Sat Jun 16 12:40:31 UTC 2012 1339850431 75864 28 -Sat Jun 16 12:41:01 UTC 2012 1339850461 75864 28 -Sat Jun 16 12:41:31 UTC 2012 1339850491 75864 28 -Sat Jun 16 12:42:01 UTC 2012 1339850521 75864 28 -Sat Jun 16 12:42:31 UTC 2012 1339850552 75864 28 -Sat Jun 16 12:43:02 UTC 2012 1339850582 75864 28 -Sat Jun 16 12:43:32 UTC 2012 1339850612 75864 28 -Sat Jun 16 12:44:02 UTC 2012 1339850642 75864 28 -Sat Jun 16 12:44:32 UTC 2012 1339850672 75864 28 -Sat Jun 16 12:45:02 UTC 2012 1339850702 75864 28 -Sat Jun 16 12:45:32 UTC 2012 1339850732 75864 28 -Sat Jun 16 12:46:02 UTC 2012 1339850762 75864 28 -Sat Jun 16 12:46:32 UTC 2012 1339850792 75864 28 -Sat Jun 16 12:47:02 UTC 2012 1339850822 75864 28 -Sat Jun 16 12:47:32 UTC 2012 1339850852 75864 28 -Sat Jun 16 12:48:02 UTC 2012 1339850882 75864 28 -Sat Jun 16 12:48:32 UTC 2012 1339850912 75864 28 -Sat Jun 16 12:49:02 UTC 2012 1339850942 75864 28 -Sat Jun 16 12:49:32 UTC 2012 1339850972 75864 28 -Sat Jun 16 12:50:02 UTC 2012 1339851002 75864 28 -Sat Jun 16 12:50:32 UTC 2012 1339851032 75864 28 -Sat Jun 16 12:51:02 UTC 2012 1339851062 75864 28 -Sat Jun 16 12:51:32 UTC 2012 1339851092 75864 28 -Sat Jun 16 12:52:02 UTC 2012 1339851122 75864 28 -Sat Jun 16 12:52:32 UTC 2012 1339851152 75864 28 -Sat Jun 16 12:53:02 UTC 2012 1339851182 75864 28 -Sat Jun 16 12:53:33 UTC 2012 1339851213 75864 28 -Sat Jun 16 12:54:03 UTC 2012 1339851243 75864 28 -Sat Jun 16 12:54:33 UTC 2012 1339851273 75864 28 -Sat Jun 16 12:55:03 UTC 2012 1339851303 75864 28 -Sat Jun 16 12:55:33 UTC 2012 1339851333 75864 28 -Sat Jun 16 12:56:03 UTC 2012 1339851363 75864 28 -Sat Jun 16 12:56:33 UTC 2012 1339851393 75864 28 -Sat Jun 16 12:57:03 UTC 2012 1339851423 75864 28 -Sat Jun 16 12:57:33 UTC 2012 1339851453 75864 28 -Sat Jun 16 12:58:03 UTC 2012 1339851483 75864 28 -Sat Jun 16 12:58:33 UTC 2012 1339851513 75864 28 -Sat Jun 16 12:59:03 UTC 2012 1339851543 75864 28 -Sat Jun 16 12:59:33 UTC 2012 1339851573 75864 28 -Sat Jun 16 13:00:03 UTC 2012 1339851603 75864 28 -Sat Jun 16 13:00:33 UTC 2012 1339851633 75864 28 -Sat Jun 16 13:01:03 UTC 2012 1339851663 75864 28 -Sat Jun 16 13:01:33 UTC 2012 1339851693 75864 28 -Sat Jun 16 13:02:03 UTC 2012 1339851723 75864 28 -Sat Jun 16 13:02:33 UTC 2012 1339851753 75864 28 -Sat Jun 16 13:03:03 UTC 2012 1339851783 75864 28 diff --git a/results/python-tornado.meminfo b/results/python-tornado.meminfo deleted file mode 100644 index 4766bbe..0000000 --- a/results/python-tornado.meminfo +++ /dev/null @@ -1,21 +0,0 @@ -Sat Jun 16 13:42:42 UTC 2012 1339854162 47172 0 -Sat Jun 16 13:43:13 UTC 2012 1339854193 197616 9813 -Sat Jun 16 13:43:43 UTC 2012 1339854223 206592 9813 -Sat Jun 16 13:44:14 UTC 2012 1339854254 214776 9813 -Sat Jun 16 13:44:44 UTC 2012 1339854284 221640 9813 -Sat Jun 16 13:45:14 UTC 2012 1339854314 227976 9813 -Sat Jun 16 13:45:45 UTC 2012 1339854345 236160 9813 -Sat Jun 16 13:46:16 UTC 2012 1339854376 242232 9813 -Sat Jun 16 13:46:46 UTC 2012 1339854406 251208 9813 -Sat Jun 16 13:47:17 UTC 2012 1339854437 259128 9813 -Sat Jun 16 13:47:47 UTC 2012 1339854467 268104 9813 -Sat Jun 16 13:48:17 UTC 2012 1339854498 274704 9813 -Sat Jun 16 13:48:48 UTC 2012 1339854528 281040 9813 -Sat Jun 16 13:49:19 UTC 2012 1339854559 289752 9813 -Sat Jun 16 13:49:49 UTC 2012 1339854589 298464 9813 -Sat Jun 16 13:50:19 UTC 2012 1339854620 305328 9813 -Sat Jun 16 13:50:50 UTC 2012 1339854650 312456 9813 -Sat Jun 16 13:51:20 UTC 2012 1339854680 322224 9813 -Sat Jun 16 13:51:51 UTC 2012 1339854711 330144 9813 -Sat Jun 16 13:52:21 UTC 2012 1339854741 337800 9813 -Sat Jun 16 13:52:51 UTC 2012 1339854771 664 0 diff --git a/results/python-ws4py.meminfo b/results/python-ws4py.meminfo deleted file mode 100644 index 8317b87..0000000 --- a/results/python-ws4py.meminfo +++ /dev/null @@ -1,27 +0,0 @@ -Sat Jun 16 13:58:30 UTC 2012 1339855110 53540 0 -Sat Jun 16 13:59:00 UTC 2012 1339855140 275300 5273 -Sat Jun 16 13:59:31 UTC 2012 1339855171 277412 5274 -Sat Jun 16 14:00:01 UTC 2012 1339855201 277412 5274 -Sat Jun 16 14:00:31 UTC 2012 1339855231 277412 5274 -Sat Jun 16 14:01:01 UTC 2012 1339855261 277412 5274 -Sat Jun 16 14:01:31 UTC 2012 1339855291 277412 5274 -Sat Jun 16 14:02:01 UTC 2012 1339855321 277412 5274 -Sat Jun 16 14:02:32 UTC 2012 1339855352 277412 5274 -Sat Jun 16 14:03:02 UTC 2012 1339855382 277412 5274 -Sat Jun 16 14:03:32 UTC 2012 1339855412 277412 5274 -Sat Jun 16 14:04:02 UTC 2012 1339855442 277412 5274 -Sat Jun 16 14:04:32 UTC 2012 1339855472 277412 5274 -Sat Jun 16 14:05:03 UTC 2012 1339855503 277412 5274 -Sat Jun 16 14:05:33 UTC 2012 1339855533 277412 5274 -Sat Jun 16 14:06:03 UTC 2012 1339855563 277412 5274 -Sat Jun 16 14:06:33 UTC 2012 1339855593 277412 5274 -Sat Jun 16 14:07:03 UTC 2012 1339855623 277412 5274 -Sat Jun 16 14:07:33 UTC 2012 1339855653 277412 5274 -Sat Jun 16 14:08:04 UTC 2012 1339855684 277412 5274 -Sat Jun 16 14:08:34 UTC 2012 1339855714 279444 108 -Sat Jun 16 14:09:04 UTC 2012 1339855744 277784 108 -Sat Jun 16 14:09:34 UTC 2012 1339855774 277784 108 -Sat Jun 16 14:10:04 UTC 2012 1339855804 277784 108 -Sat Jun 16 14:10:34 UTC 2012 1339855834 277784 108 -Sat Jun 16 14:11:04 UTC 2012 1339855864 277784 108 -Sat Jun 16 14:11:34 UTC 2012 1339855894 277784 108 diff --git a/results/ruby-em-websockets.meminfo b/results/ruby-em-websockets.meminfo deleted file mode 100644 index ed30a7c..0000000 --- a/results/ruby-em-websockets.meminfo +++ /dev/null @@ -1,3 +0,0 @@ -Sat Jun 16 09:43:06 UTC 2012 1339839786 5896 0 -Sat Jun 16 09:43:36 UTC 2012 1339839816 5896 1000 -Sat Jun 16 09:44:07 UTC 2012 1339839847 5896 1000 diff --git a/runtest b/runtest index 53cacd3..4ebfb66 100755 --- a/runtest +++ b/runtest @@ -4,7 +4,9 @@ -mode(compile). -main([LogFile, SecondsStr, Server, PortStr, ClientsStr]) -> +main([LogFile, ClientsStr, SecondsStr]) -> + main([LogFile, "localhost", "8000", ClientsStr, SecondsStr]); +main([LogFile, Server, PortStr, ClientsStr, SecondsStr]) -> Port = list_to_integer(PortStr), Clients = list_to_integer(ClientsStr), Seconds = list_to_integer(SecondsStr), @@ -18,23 +20,28 @@ main([LogFile, SecondsStr, Server, PortStr, ClientsStr]) -> [Clients, Seconds, LogFile]), {ok, _} = wsdemo_logger:start_link(LogFile), - spawn(fun() -> report_loop(erlang:now(), Seconds) end), - {ok, _} = wsdemo_stats:start_link(Server, Port, Clients), - io:format("Running~n", []), - timer:sleep(timer:seconds(Seconds)), - io:format("Finished~n", []), - wsdemo_logger:close(), - ok; -main([LogFile, SecondsStr, ClientsStr]) -> - main([LogFile, SecondsStr, "localhost", "8000", ClientsStr]); + + spawn(fun() -> + report_loop(now(), Seconds) + end), + erlang:send_after(timer:seconds(Seconds), self(), done), + + receive + done -> + wsdemo_stats:stop(), + wsdemo_logger:close(), + done + end; + main(_) -> usage(). + usage() -> io:format("usage: ~n" - "runtest LogFile::string() Seconds::integer() ClientCount::integer() | ~n" - "runtest LogFile::string() Seconds::integer() Server::string() Port::integer() ClientCount::integer()~n", []). + "runtest LogFile::string() ClientCount::integer() Seconds::integer() | ~n" + "runtest LogFile::string() Server::string() Port::integer() ClientCount::integer() Seconds::integer()~n", []). report_loop(Start, Seconds) -> case whereis(wsdemo_logger) of diff --git a/src/websocket_client.erl b/src/websocket_client.erl index ada7292..a325133 100644 --- a/src/websocket_client.erl +++ b/src/websocket_client.erl @@ -17,7 +17,8 @@ -behaviour(gen_server). %% API --export([start_link/3,start_link/4,write/2,write_sync/2,close/1, frame/1, websocket_mask/2]). +-export([start_link/3, start_link/4, stop/1, + write/2, write_sync/2, close/1, frame/1]). %% gen_server callbacks -export([init/1, handle_call/3, handle_cast/2, handle_info/2, @@ -46,10 +47,8 @@ start_link(Mod,Host,Port) -> start_link(Mod,Host,Port,Path) -> gen_server:start_link(?MODULE, [Mod,Host,Port,Path], []). -init([Mod,Host,Port,Path]) -> - ModState = Mod:ws_init(), - {ok, #state{server={Host,Port,Path}, - callback=Mod, callback_state=ModState}, 0}. +stop(Pid) -> + gen_server:call(Pid, stop). %% Write to the server write(Pid, Data) -> @@ -63,6 +62,11 @@ write_sync(ClientState, Data) -> close(Pid) -> gen_server:cast(Pid,close). +init([Mod,Host,Port,Path]) -> + ModState = Mod:ws_init(), + {ok, #state{server={Host,Port,Path}, + callback=Mod, callback_state=ModState}, 0}. + handle_cast({send,Data}, State) -> write_sync(State, Data), {noreply, State}; @@ -149,6 +153,9 @@ handle_info(Msg, State) -> CBState = Mod:ws_info(State, Msg, State#state.callback_state), {noreply, State#state{callback_state=CBState}}. +handle_call(stop, _From, State) -> + gen_tcp:close(State#state.socket), + {stop, normal, ok, State}; handle_call(_Request,_From,State) -> {reply,ok,State}. diff --git a/src/wsdemo.app.src b/src/wsdemo.app.src deleted file mode 100644 index 24ab168..0000000 --- a/src/wsdemo.app.src +++ /dev/null @@ -1,12 +0,0 @@ -{application, wsdemo, - [ - {description, ""}, - {vsn, "1"}, - {registered, []}, - {applications, [ - kernel, - stdlib - ]}, - {mod, { wsdemo_app, []}}, - {env, []} - ]}. diff --git a/src/wsdemo_bench.app.src b/src/wsdemo_bench.app.src new file mode 100644 index 0000000..675522f --- /dev/null +++ b/src/wsdemo_bench.app.src @@ -0,0 +1,39 @@ +% -*- erlang -*- + +{application, wsdemo_bench, + [ + {description, ""}, + {vsn, "1"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {mod, { wsdemo_bench_app, []}}, + {env, [ + {supervisord, {"localhost", 9001}}, + {clients, 10000}, + {seconds, 300}, + {servers, [ + "clojure-aleph", + "erlang-cowboy", + "go-gonet", + "haskell-snap", + "java-webbit", + "node-ws", + "node-ws-cluster", + "perl-ev", + "pypy-tornado-1", + "pypy-tornado-N", + "pypy-twisted-1", + "pypy-twisted-N", + "python-gevent-websocket-1", + "python-gevent-websocket-N", + "python-tornado-1", + "python-tornado-N", + "python-twisted-1", + "python-twisted-N", + "scala-play" + ]} + ]} + ]}. diff --git a/src/wsdemo_bench.erl b/src/wsdemo_bench.erl new file mode 100644 index 0000000..f9a52ce --- /dev/null +++ b/src/wsdemo_bench.erl @@ -0,0 +1,43 @@ +-module(wsdemo_bench). + +-export([start/0, run_sync/0]). + +start() -> + application:start(wsdemo_bench). + +run_sync() -> + Self = self(), + Ref = make_ref(), + + CB = fun(Reason) -> + Self ! {Ref, Reason} + end, + + [Servers, DBRoot, Host, Port, Clients, Seconds] = + get_keys_or_not([servers, db_root, host, port, clients, seconds]), + + wsdemo_master_fsm:run_suite(CB, {Servers, DBRoot, + Host, Port, Clients, Seconds}), + % wait for the suite to finish. + receive + {Ref, Reason} -> + Reason + end. + +get_keys_or_not(Keys) -> + get_keys_or_not([], Keys). + +get_keys_or_not(Acc,[]) -> + lists:reverse(Acc); +get_keys_or_not(Acc,[Key|Rest]) -> + case application:get_env(wsdemo_bench, Key) of + {ok, Val} -> + get_keys_or_not([Val|Acc], Rest); + undefined -> + {error, {application_env, Key, not_found}} + end. + + + + + diff --git a/src/wsdemo_app.erl b/src/wsdemo_bench_app.erl similarity index 73% rename from src/wsdemo_app.erl rename to src/wsdemo_bench_app.erl index d81028d..1362392 100644 --- a/src/wsdemo_app.erl +++ b/src/wsdemo_bench_app.erl @@ -1,4 +1,4 @@ --module(wsdemo_app). +-module(wsdemo_bench_app). -behaviour(application). @@ -10,7 +10,9 @@ %% =================================================================== start(_StartType, _StartArgs) -> - wsdemo_sup:start_link(). + % complete the cluster if the app var exists + wsdemo_bench_sup:start_link(). stop(_State) -> ok. + diff --git a/src/wsdemo_sup.erl b/src/wsdemo_bench_sup.erl similarity index 72% rename from src/wsdemo_sup.erl rename to src/wsdemo_bench_sup.erl index 30e9fb3..05704a3 100644 --- a/src/wsdemo_sup.erl +++ b/src/wsdemo_bench_sup.erl @@ -1,5 +1,5 @@ --module(wsdemo_sup). +-module(wsdemo_bench_sup). -behaviour(supervisor). @@ -24,5 +24,8 @@ start_link() -> %% =================================================================== init([]) -> - {ok, { {one_for_one, 5, 10}, []} }. + {ok, { {one_for_one, 5, 60}, [ + ?CHILD(wsdemo_server_manager, worker), + ?CHILD(wsdemo_master_fsm, worker) + ]} }. diff --git a/src/wsdemo_master_fsm.erl b/src/wsdemo_master_fsm.erl new file mode 100644 index 0000000..13f1d50 --- /dev/null +++ b/src/wsdemo_master_fsm.erl @@ -0,0 +1,170 @@ +-module(wsdemo_master_fsm). +-behaviour(gen_fsm). +-define(SERVER, ?MODULE). +-record(state, {config, servers, callback, runner}). + +%% ------------------------------------------------------------------ +%% API Function Exports +%% ------------------------------------------------------------------ + +-export([start_link/0, run_suite/2, run_suite/3, run_suite/6]). + +%% ------------------------------------------------------------------ +%% gen_fsm Function Exports +%% ------------------------------------------------------------------ + +-export([init/1, handle_event/3, + handle_sync_event/4, handle_info/3, terminate/3, + code_change/4]). + +%% state handlers +-export([idle/2, warmup/2, fulltest/2]). + +%% ------------------------------------------------------------------ +%% API Function Definitions +%% ------------------------------------------------------------------ + +start_link() -> + gen_fsm:start_link({local, ?SERVER}, ?MODULE, [], []). + +run_suite(Servers, Clients, Seconds) -> + run_suite(Servers, "/tmp", "127.0.0.1", 8000, Clients, Seconds). + +run_suite(Servers, DataRoot, Host, Port, Clients, Seconds) -> + Callback = fun(R) -> + error_logger:info_msg("~w~n", [R]) + end, + run_suite(Callback, {Servers, DataRoot, Host, Port, Clients, Seconds}). + +run_suite(Callback, {_Servers, _DataRoot, _Host, _Port, _Clients, _Seconds} = Config) -> + gen_fsm:send_event(?SERVER, {run_suite, Callback, Config}). + +%% ------------------------------------------------------------------ +%% gen_fsm Function Definitions +%% ------------------------------------------------------------------ + +init(_Args) -> + process_flag(trap_exit, true), + {ok, idle, #state{}}. + +idle({run_suite, Callback, Config}, State) -> + error_logger:info_msg("Running suite ~p~n", [config_to_propslist(Config)]), + Servers = element(1, Config), + State2 = State#state{callback=Callback, + servers=Servers, + config=Config}, + + handle_event(run_warmup, idle, State2). + +warmup(run_fulltest, State) -> + handle_event(run_fulltest, warmup, State). + +fulltest(next_server, State) -> + handle_event(next_server, fulltest, State). + +handle_event(run_warmup, StateName, State) -> + case do_warmup(State) of + {ok, State2} -> + {next_state, warmup, State2}; + _Error -> + % could not start the server, move on + handle_event(cooldown, StateName, State) + end; +handle_event(run_fulltest, _StateName, State) -> + State2 = do_fulltest(State), + {next_state, fulltest, State2}; +handle_event(cooldown, _StateName, State) -> + % stop the current server + wsdemo_server_manager:stop_server(current_server(State)), + + % schedule the end of the cooldown phase + erlang:send_after(timer:seconds(15), self(), next_server), + + error_logger:info_msg("Stopping ~s and cooling down for 15s~n", [current_server(State)]), + {next_state, cooldown, State}; +handle_event(next_server, StateName, #state{callback=CB, servers=[_|Rest]} = State) -> + case Rest of + [] -> + CB(done), + % we are out of servers to test, return to the idle state + {next_state, idle, #state{}}; + Rest -> + % pop off the current server and move to the warm up phase + % for the next server + handle_event(run_warmup, StateName, State#state{servers=Rest}) + end. + +handle_sync_event(Event, From, StateName, _State) -> + % crash an unknown event + exit({error, {invalid_event, {Event, From, StateName}}}). + + +% In 'warmup', a normal exit of the runner proc moves us to 'fulltest' +handle_info({'EXIT', Pid, normal}, warmup, #state{runner=Pid} = State) -> + warmup(run_fulltest, State); +% In 'fulltest', a normal exit of the runner proc moves us to 'cooldown' +handle_info({'EXIT', Pid, normal}, fulltest, #state{runner=Pid} = State) -> + handle_event(cooldown, fulltest, State); +% In any phase, an abnormal exit of the runner proc moves us to 'cooldown' +handle_info({'EXIT', Pid, _Reason}, StateName, #state{runner=Pid} = State) -> + handle_event(cooldown, StateName, State); +% A next_server message is sent when the cooldown is over +handle_info(next_server, cooldown, State) -> + % skip the test if the runner crashes + handle_event(next_server, cooldown, State). + +terminate(_Reason, _StateName, _State) -> + ok. + +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +%% ------------------------------------------------------------------ +%% Internal Function Definitions +%% ------------------------------------------------------------------ +current_server(#state{servers=[Current|_]}) -> + Current. + +config_to_propslist({Servers, DataRoot, Host, Port, Clients, Seconds}) -> + [{servers, Servers}, + {db_root, DataRoot}, + {host, Host}, + {port, Port}, + {clients, Clients}, + {seconds, Seconds}]. + +do_atest(DBName, Modifier, State) -> + Server = current_server(State), + {_, DBRoot, Host, Port, Clients, Seconds} = State#state.config, + DB = filename:join(DBRoot, DBName), + + Clients2 = trunc(Clients * Modifier), + Seconds2 = trunc(Seconds * Modifier), + + error_logger:info_msg("Testing ~p~n", [[{server, Server}, + {db, DB}, + {host, Host}, + {port, Port}, + {clients, Clients2}, + {seconds, Seconds2}]]), + + {ok, Pid} = wsdemo_runner_fsm:start_link(Server, DB, Host, Port, Clients, Seconds2), + State#state{runner=Pid}. + +do_warmup(State) -> + Server = current_server(State), + + error_logger:info_msg("Starting ~s and waiting 15s for server init~n", [Server]), + case wsdemo_server_manager:start_server(Server) of + ok -> + timer:sleep(timer:seconds(15)), + {ok, do_atest("warmup", 0.1, State)}; + Error -> + error_logger:error_msg("Cound not start server ~s, moving onto next server~nReason: ~p~n", [Server, Error]), + Error + end. + + +do_fulltest(State) -> + [Server|_] = State#state.servers, + do_atest(Server, 1, State). diff --git a/src/wsdemo_runner_fsm.erl b/src/wsdemo_runner_fsm.erl new file mode 100644 index 0000000..b104c18 --- /dev/null +++ b/src/wsdemo_runner_fsm.erl @@ -0,0 +1,95 @@ +-module(wsdemo_runner_fsm). +-behaviour(gen_fsm). +-define(SERVER, ?MODULE). +-record(state, {server_name, db, host, port, clients, seconds}). + +%% ------------------------------------------------------------------ +%% API Function Exports +%% ------------------------------------------------------------------ + +-export([start_link/6, cancel/0]). + +%% ------------------------------------------------------------------ +%% gen_fsm Function Exports +%% ------------------------------------------------------------------ + +-export([init/1, handle_event/3, + handle_sync_event/4, handle_info/3, terminate/3, + code_change/4]). + +% event handlers +-export([running/3]). + +%% ------------------------------------------------------------------ +%% API Function Definitions +%% ------------------------------------------------------------------ + +start_link(ServerName, DB, Host, Port, Clients, Seconds) -> + gen_fsm:start_link({local, ?SERVER}, ?MODULE, + [ServerName, DB, Host, Port, Clients, Seconds], []). + +cancel() -> + gen_fsm:sync_send_event(?SERVER, cancel). + +%% ------------------------------------------------------------------ +%% gen_fsm Function Definitions +%% ------------------------------------------------------------------ + +init([ServerName, DB, Host, Port, Clients, Seconds]) -> + State = #state{server_name=ServerName, + db=DB, + host=Host, + port=Port, + clients=Clients, + seconds=Seconds}, + State2 = start_test(State), + {ok, running, State2}. + +running(cancel, _, State) -> + State2 = stop_test(State), + {stop, ok, cancel, State2}. + +handle_event(Event, StateName, _State) -> + % crash an unknown event + exit({error, {invalid_event, {Event, StateName}}}). + +handle_sync_event(Event, From, StateName, _State) -> + % crash an unknown sync event + exit({error, {invalid_sync_event, {Event, From, StateName}}}). + +handle_info(timer_done, running, State) -> + stop_test(State), + {stop, normal, #state{}}. + +terminate(_Reason, _StateName, _State) -> + ok. + +code_change(_OldVsn, StateName, State, _Extra) -> + {ok, StateName, State}. + +%% ------------------------------------------------------------------ +%% Internal Function Definitions +%% ------------------------------------------------------------------ + +start_test(State) -> + error_logger:info_msg("Testing ~p~n", [State]), + {ok, _} = wsdemo_logger:start_link(State#state.db), + {ok, _} = wsdemo_server_logger:start_link(State#state.host, + State#state.port, + State#state.server_name), + {ok, _} = wsdemo_stats:start_link(State#state.host, + State#state.port, + State#state.clients), + + erlang:send_after(timer:seconds(State#state.seconds), + self(), + timer_done), + % TODO: Run the test + State. + +stop_test(State) -> + ok = wsdemo_stats:stop(), + ok = wsdemo_server_logger:stop(), + ok = wsdemo_logger:close(), + % TODO: Stop the test + State. diff --git a/src/wsdemo_server_logger.erl b/src/wsdemo_server_logger.erl new file mode 100644 index 0000000..ea234ca --- /dev/null +++ b/src/wsdemo_server_logger.erl @@ -0,0 +1,67 @@ +-module(wsdemo_server_logger). +-behaviour(gen_server). +-define(SERVER, ?MODULE). + +%% ------------------------------------------------------------------ +%% API Function Exports +%% ------------------------------------------------------------------ + +-export([start_link/3, start_link/2, stop/0]). + +%% ------------------------------------------------------------------ +%% gen_server Function Exports +%% ------------------------------------------------------------------ + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +%% ------------------------------------------------------------------ +%% API Function Definitions +%% ------------------------------------------------------------------ +start_link(Host, Port, Servername) when is_integer(Port) -> + HostAndPort = lists:flatten([Host, ":", integer_to_list(Port)]), + start_link(HostAndPort, Servername). + +-spec start_link(iolist(), string()) -> {ok, pid()}. +start_link(HostAndPort, Servername) -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [HostAndPort, Servername], []). + +stop() -> + gen_server:call(?SERVER, stop). + +%% ------------------------------------------------------------------ +%% gen_server Function Definitions +%% ------------------------------------------------------------------ + +init([HostAndPort, Servername]) -> + erlang:send_after(timer:seconds(1), self(), log_stats), + {ok, {HostAndPort, Servername}}. + +handle_call(stop, _From, State) -> + {stop, normal, ok, State}; +handle_call(_Request, _From, State) -> + {reply, ok, State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(log_stats, {HostAndPort, Servername}) -> + {ok, RSS} = wsdemo_server_manager:memusage(Servername), + {ok, Connections} = wsdemo_server_manager:connections(HostAndPort), + + wsdemo_logger:event({server, {rss, RSS}}), + wsdemo_logger:event({server, {connections, Connections}}), + + erlang:send_after(timer:seconds(1), self(), log_stats), + {noreply, {HostAndPort, Servername}}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% ------------------------------------------------------------------ +%% Internal Function Definitions +%% ------------------------------------------------------------------ + diff --git a/src/wsdemo_server_manager.erl b/src/wsdemo_server_manager.erl new file mode 100644 index 0000000..011bb1b --- /dev/null +++ b/src/wsdemo_server_manager.erl @@ -0,0 +1,166 @@ +-module(wsdemo_server_manager). +-behaviour(gen_server). +-define(SERVER, ?MODULE). +-record(state, {host, port}). +%% ------------------------------------------------------------------ +%% API Function Exports +%% ------------------------------------------------------------------ + +-export([start_link/0, start_link/1, start_server/1, stop_server/1, + list_servers/0, memusage/1, connections/1, status/1]). + +%% ------------------------------------------------------------------ +%% gen_server Function Exports +%% ------------------------------------------------------------------ + +-export([init/1, handle_call/3, handle_cast/2, handle_info/2, + terminate/2, code_change/3]). + +%% ------------------------------------------------------------------ +%% API Function Definitions +%% ------------------------------------------------------------------ +-type error() :: {error, any()}. +-type status() :: stopped | starting | running | backoff | stopping | exited | fatal | unknown. +-type proplists() :: [{Key1 :: any(), Value1 :: any()}]. + +-spec start_link() -> {ok, pid()} | error(). +start_link() -> + {ok, Supervisord} = application:get_env(wsdemo_bench, supervisord), + start_link(Supervisord). + +-spec start_link({Host :: string(), Port :: integer()}) -> {ok, pid()} | error(). +start_link(Supervisord) -> + gen_server:start_link({local, ?SERVER}, ?MODULE, [Supervisord], []). + +-spec start_server(string()) -> ok | error(). +start_server(ServerName) -> + gen_server:call(?SERVER, {start_server, ServerName}). + +-spec stop_server(string()) -> ok | error(). +stop_server(ServerName) -> + gen_server:call(?SERVER, {stop_server, ServerName}). + +-spec status(string()) -> {ok, status()} | error(). +status(ServerName) -> + gen_server:call(?SERVER, {status, ServerName}). + +-spec memusage(string()) -> {ok, integer()} | error(). +memusage(ServerName) -> + gen_server:call(?SERVER, {memusage, ServerName}). + +-spec connections(HostAndPort :: iolist()) -> {ok, integer()} | error(). +connections(HostAndPort) -> + gen_server:call(?SERVER, {connections, HostAndPort}). + +-spec list_servers() -> {ok, [{ServerName :: string(), proplists()}]} | error(). +list_servers() -> + gen_server:call(?SERVER, list_servers). + +%% ------------------------------------------------------------------ +%% gen_server Function Definitions +%% ------------------------------------------------------------------ + +init([{Host, Port}]) -> + {ok, #state{host=Host, port=Port}}. + +handle_call(list_servers, _From, State) -> + {reply, rpc_list_servers(State), State}; +handle_call({start_server, ServerName}, _From, State) -> + {reply, rpc_start_server(ServerName, State), State}; +handle_call({stop_server, ServerName}, _From, State) -> + {reply, rpc_stop_server(ServerName, State), State}; +handle_call({status, ServerName}, _From, State) -> + {reply, rpc_status(ServerName, State), State}; +handle_call({memusage, ServerName}, _From, State) -> + {reply, rpc_call('wsdemo_monitor.memusage', [ServerName], State), State}; +handle_call({connections, HostAndPort}, _From, State) -> + {reply, rpc_call('wsdemo_monitor.connections', [HostAndPort], State), State}. + +handle_cast(_Msg, State) -> + {noreply, State}. + +handle_info(_Msg, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + +%% ------------------------------------------------------------------ +%% Internal Function Definitions +%% ------------------------------------------------------------------ +supervisord_status(Status) -> +%https://github.com/Supervisor/supervisor/blob/master/supervisor/states.py#L4 +%class ProcessStates: +% STOPPED = 0 +% STARTING = 10 +% RUNNING = 20 +% BACKOFF = 30 +% STOPPING = 40 +% EXITED = 100 +% FATAL = 200 +% UNKNOWN = 1000 + case Status of + 0 -> stopped; + 10 -> starting; + 20 -> running; + 30 -> backoff; + 40 -> stopping; + 100 -> exited; + 200 -> fatal; + 1000 -> unknown + end. + +rpc_status(ServerName, State) -> + case rpc_call('supervisor.getProcessInfo', [ServerName], State) of + {ok, {struct, Props}} -> + supervisord_status(proplists:get_value(state, Props)); + Error -> + Error + end. + +procinfo_to_server_list({array, ProcInfoList}) -> + procinfo_to_server_list(ProcInfoList, []). + +procinfo_to_server_list([{struct, ProcInfo}|Rest], Accum) -> + ServerName = proplists:get_value(name, ProcInfo), + Status = supervisord_status(proplists:get_value(state, ProcInfo)), + procinfo_to_server_list(Rest, [{ServerName, Status}|Accum]); +procinfo_to_server_list([], Accum) -> + lists:reverse(Accum). + +rpc_list_servers(State) -> + case rpc_call('supervisor.getAllProcessInfo', [], State) of + {ok, ProcessInfo} -> + {ok, procinfo_to_server_list(ProcessInfo)}; + Error -> + Error + end. +rpc_start_server(ServerName, State) -> + case rpc_call('supervisor.startProcess', [ServerName], State) of + {ok, true} -> + ok; + Error -> + Error + end. + +rpc_stop_server(ServerName, State) -> + case rpc_call('supervisor.stopProcess', [ServerName], State) of + {ok, true} -> + ok; + Error -> + Error + end. + +rpc_call(Method, Args, #state{host=Host, port=Port}) -> + case xmlrpc:call(Host, Port, "/RPC2", {call, Method, Args}) of + {ok,{response,[Return]}} -> + {ok, Return}; + {ok,{response,{fault,Code,Reason}}} -> + {error, {fault, Code, Reason}}; + {error, Reason} -> + {error, Reason} + end. + diff --git a/src/wsdemo_stats.erl b/src/wsdemo_stats.erl index b76b6b0..fa27c75 100644 --- a/src/wsdemo_stats.erl +++ b/src/wsdemo_stats.erl @@ -37,7 +37,7 @@ init([Hostname, Port, Clients]) -> handle_call(stop, _From, State) -> lists:foreach(fun(Pid) -> - websocket_client:close(Pid) + catch websocket_client:stop(Pid) end, State#state.clients), {stop, normal, ok, State}. diff --git a/stats/Makefile b/stats/Makefile index 0f8f4d4..b99e14a 100644 --- a/stats/Makefile +++ b/stats/Makefile @@ -2,25 +2,30 @@ all: stats -stats: dirs counts.csv latencies.csv handshake.csv +inject-data: + @echo "---- Creating Database" + createdb wsdemo + @echo "---- Installing the DB Schema" + psql < database_setup.sql + @echo "---- Injecting raw data (expect this to take a lot of time" + psql < inject.sql + @echo "---- Aggregating data" + psql < aggregate.sql + +drop-data: + dropdb wsdemo + +stats: dirs counts.csv R CMD BATCH stats.r dirs: mkdir -p stat_results clean: - rm -f counts.csv latencies.csv handshake.csv + rm -f counts.csv rm -f *.Rout rm -f stat_results/*.pdf counts.csv: echo "framework, clients, handshakes, connection_timeouts, messages_sent, messages_recv, crashes" > counts.csv for k in ../data/*/counts.csv; do tail -n 1 "$$k" >> counts.csv ; done - -latencies.csv: - echo "framework, timestamp, elapsed" > latencies.csv - for k in ../data/*/message_latencies.csv; do sed -e '1d' "$$k" >> latencies.csv; done - -handshake.csv: - echo "framework, timestamp, elapsed" > handshake.csv - for k in ../data/*/handshake_times.csv; do sed -e '1d' "$$k" >> handshake.csv; done diff --git a/stats/aggregate.sql b/stats/aggregate.sql new file mode 100644 index 0000000..fcc99d2 --- /dev/null +++ b/stats/aggregate.sql @@ -0,0 +1,5 @@ +\c wsdemo + +INSERT INTO latencies_small + SELECT * FROM latencies_skew + WHERE random() < 0.01; diff --git a/stats/common.r b/stats/common.r new file mode 100644 index 0000000..8673d99 --- /dev/null +++ b/stats/common.r @@ -0,0 +1,25 @@ +# R-script for producing statistics output: + +# Load all the necessary packages, installing missing ones when necessary +packages.to.install <- c("plyr", "ggplot2", "RPostgreSQL", "quantreg") + +for(p in packages.to.install) { + print(p) + if (suppressWarnings(!require(p, character.only = TRUE))) install.packages(p, repos = "http://lib.stat.cmu.edu/R/CRAN") + if (p == "ggplot2") suppressWarnings(library(ggplot2)) +} + +# Make sure needed libraries are there +require(ggplot2) +require(quantreg) +library(RPostgreSQL) + +con <- dbConnect(dbDriver("PostgreSQL"), dbname = 'wsdemo') + +read_times <- function(dbTable) { + handshake <- dbReadTable(con, dbTable) + transform(handshake, + elapsed_ms = elapsed / 1000, + timestamp_ms = timestamp / 1000, + timestamp_s = timestamp / 1000000) +} diff --git a/stats/database_setup.sql b/stats/database_setup.sql new file mode 100644 index 0000000..3d2522a --- /dev/null +++ b/stats/database_setup.sql @@ -0,0 +1,57 @@ +-- Database setupfor for a PostgreSQL database +\c wsdemo + +-- Handshake data +CREATE TABLE handshakes ( + framework VARCHAR(30), + timestamp BIGINT, + elapsed BIGINT +); + +CREATE VIEW handshakes_min AS + SELECT framework, min(timestamp) AS min_timestamp + FROM handshakes + GROUP BY framework; + +CREATE OR REPLACE VIEW handshakes_skew AS + SELECT l.framework as framework + , l.timestamp - lm.min_timestamp as timestamp + , l.elapsed as elapsed + FROM handshakes l INNER JOIN handshakes_min lm ON (l.framework = lm.framework); + +CREATE TABLE latencies ( + framework VARCHAR(30), + timestamp BIGINT, + elapsed BIGINT +); + +CREATE VIEW latencies_min AS + SELECT framework, min(timestamp) as min_timestamp + FROM latencies + GROUP BY framework; + +CREATE OR REPLACE VIEW latencies_skew AS + SELECT l.framework as framework + , l.timestamp - lm.min_timestamp as timestamp + , l.elapsed as elapsed + FROM latencies l INNER JOIN latencies_min lm ON (l.framework = lm.framework); + +CREATE VIEW latencies_sample AS + SELECT * + FROM latencies + WHERE random() < 0.01; + +CREATE TABLE latencies_small ( + framework VARCHAR(30), + timestamp BIGINT, + elapsed BIGINT +); + +CREATE TABLE events ( + timestamp BIGINT, + framework VARCHAR(30), + client_id VARCHAR(20), + event_key VARCHAR(20), + event_data TEXT +); + diff --git a/stats/inject.sql b/stats/inject.sql new file mode 100644 index 0000000..bb8a22c --- /dev/null +++ b/stats/inject.sql @@ -0,0 +1,78 @@ +\c wsdemo + +-- Copy all data into pg + +-- Events +--\COPY events FROM '../data/vagrant-c1k/clojure-aleph/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/erlang-cowboy/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/go-gonet/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/haskell-snap/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/java-webbit/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/node-ws/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/node-ws-cluster/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/perl-ev/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/pypy-tornado-1/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/pypy-tornado-N/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/pypy-twisted-1/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/pypy-twisted-N/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/python-tornado-1/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/python-tornado-N/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/python-twisted-1/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/python-twisted-N/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/python-gevent-websocket-1/events.csv' WITH CSV HEADER +--\COPY events FROM '../data/vagrant-c1k/python-gevent-websocket-N/events.csv' WITH CSV HEADER + +-- Latencies +\COPY latencies FROM '../data/clojure-aleph/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/erlang-cowboy/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/go-gonet/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/haskell-snap/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/java-webbit/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/node-ws/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/node-ws-cluster/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/perl-ev/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/pypy-tornado-1/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/pypy-tornado-N/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/pypy-twisted-1/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/pypy-twisted-N/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/python-tornado-1/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/python-tornado-N/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/python-twisted-1/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/python-twisted-N/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/python-gevent-websocket-1/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/python-gevent-websocket-N/message_latencies.csv' WITH CSV HEADER +\COPY latencies FROM '../data/scala-play/message_latencies.csv' WITH CSV HEADER + +-- Handshakes +\COPY handshakes FROM '../data/clojure-aleph/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/erlang-cowboy/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/go-gonet/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/haskell-snap/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/java-webbit/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/node-ws/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/node-ws-cluster/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/perl-ev/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/pypy-tornado-1/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/pypy-tornado-N/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/pypy-twisted-1/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/pypy-twisted-N/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/python-tornado-1/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/python-tornado-N/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/python-twisted-1/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/python-twisted-N/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/python-gevent-websocket-1/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/python-gevent-websocket-N/handshake_times.csv' WITH CSV HEADER +\COPY handshakes FROM '../data/scala-play/handshake_times.csv' WITH CSV HEADER + + +-- Hand the tables a couple of indexes +-- This speeds up queries considerably since there is stuff to hook onto +-- for the query engine. +CREATE INDEX handshakes_ts_idx ON handshakes(timestamp); +CREATE INDEX handshakes_elapsed_idx ON handshakes(elapsed); + +CREATE INDEX latencies_ts_idx ON latencies(timestamp); +CREATE INDEX latencies_elapsed_idx ON latencies(elapsed); +CREATE INDEX latencies_fwrk ON latencies(framework); + +CREATE INDEX events_ts_idx ON events(timestamp); diff --git a/stats/stats.r b/stats/stats.r index 8db2f80..1710058 100644 --- a/stats/stats.r +++ b/stats/stats.r @@ -1,7 +1,7 @@ # R-script for producing statistics output: # Load all the necessary packages, installing missing ones when necessary -packages.to.install <- c("plyr", "ggplot2") +packages.to.install <- c("plyr", "ggplot2", "RPostgreSQL") for(p in packages.to.install) { print(p) @@ -11,16 +11,29 @@ for(p in packages.to.install) { # Make sure needed libraries are there require(ggplot2) +library(RPostgreSQL) # Set some global variables base_size <- 9 # Read in data +con <- dbConnect(dbDriver("PostgreSQL"), dbname = 'wsdemo') + counts <- read.csv("counts.csv", header=TRUE) -handshake <- read.csv("handshake.csv", header=TRUE) +handshake <- dbReadTable(con, "handshakes_skew") handshake <- transform(handshake, elapsed_ms = elapsed / 1000) -#latencies <- read.csv("latencies.csv", header=TRUE) +latencies <- dbReadTable(con, "latencies_small") +latencies <- transform(latencies, elapsed_ms = elapsed / 1000) + +options <- opts(axis.text.x = theme_text(size = base_size * 0.8, + angle = 330, + hjust = 0, + colour = "grey50"), + axis.text.x = theme_text(size = base_size * 0.5), + axis.text.y = theme_text(size = base_size * 0.5), + strip.text.x = theme_text(size = base_size * 0.5)) + # A plot of the connection timeouts as a function ws.plot.conn_timeout <- function() { @@ -29,84 +42,50 @@ ws.plot.conn_timeout <- function() { + geom_bar() + xlab('Framework') + ylab('Connections Lost') - + opts(axis.ticks = theme_blank(), - axis.text.x = theme_text(size = base_size * 0.8, - angle = 330, - hjust = 0, - colour = "grey50"))) + + options) } -# Jitter plot of the handshake times -ws.plot.jitter <- function(T) { - box <- ggplot(T, aes(factor(framework), elapsed_ms)) - (box + geom_jitter(alpha = 0.08) + coord_trans(y = "log10") - + xlab('Framework') - + ylab('Handshake Time (ms)') - + opts(axis.ticks = theme_blank(), - axis.text.x = theme_text(size = base_size * 0.8, - angle = 330, - hjust = 0, - colour = "grey50"))) +ws.plot.bin2d <- function(T, YLab) { + dots <- ggplot(T, aes(x=timestamp/1000000, y=elapsed/1000)) + (dots + + facet_wrap(~ framework) + + ylab(YLab) + + xlab('Second') + + stat_bin2d() + + options) } -ws.plot.box <- function(T) { - box <- ggplot(T, aes(x = factor(framework), y = elapsed_ms)) - (box + geom_boxplot() + coord_trans(y = "log10") +ws.plot.box <- function(T, Y) { + box <- ggplot(T, aes(x = factor(framework), y = elapsed/1000)) + (box + + geom_boxplot(outlier.shape = NA, alpha=0.5) + + scale_y_log10() + xlab('Framework') - + ylab('Handshake Time (ms)') - + opts(axis.text.x = theme_text(size = base_size * 0.8, - angle = 330, - hjust = 0, - colour = "grey50"))) + + ylab(Y) + + options) } -# Volcano -ws.plot.volcano <- function(T) { - v <- ggplot(T, aes(x = log10(elapsed_ms))) - (v + stat_density(aes(ymax = ..density.., ymin = -..density..), - fill = "grey50", - colour = "grey50", - geom = "ribbon", - position = "identity") - + facet_grid(. ~ framework) - + coord_flip()) +ws.plot.histogram <- function(T, Xlab) { + x <- ggplot(T, aes(x = elapsed_ms)) + (x + geom_histogram() + + xlab(Xlab) + + scale_x_log10() + + facet_wrap(~ framework) + + options) } ## BEGIN PLOTTING -# Conn timeout -pdf("stat_results/conn_timeouts.pdf") -ws.plot.conn_timeout() -dev.off() -png("stat_results/conn_timeouts.png") +pdf("stat_results/results.pdf") + ws.plot.conn_timeout() -dev.off() -# Handshake Jitter -pdf("stat_results/handshake_jitter.pdf") -ws.plot.jitter(handshake) -dev.off() -png("stat_results/handshake_jitter.png") -ws.plot.jitter(handshake) -dev.off() +ws.plot.box(handshake, 'Handshake time (ms)') +ws.plot.histogram(handshake, 'Handshake time (ms)') +ws.plot.bin2d(handshake, 'Handshake time (ms)') -# Handshake Volcano -pdf("stat_results/handshake_volcano.pdf", width=15) -ws.plot.volcano(handshake) -dev.off() +ws.plot.box(latencies, 'Message latency (ms)') +ws.plot.histogram(latencies, 'Message latency (ms)') +ws.plot.bin2d(latencies, 'Message latency (ms)') -# Handshake Box -pdf("stat_results/handshake_box.pdf") -ws.plot.box(handshake) -dev.off() -png("stat_results/handshake_box.png") -ws.plot.box(handshake) dev.off() -# Latencies Box -# pdf("stat_results/latencies_box.pdf") -# ws.plot.box(latencies) -# dev.off() -# png("stat_results/latencies_box.png") -# ws.plot.box(latencies) -# dev.off() -