diff --git a/include/eredis.hrl b/include/eredis.hrl index c97e2cd..2d89930 100644 --- a/include/eredis.hrl +++ b/include/eredis.hrl @@ -25,8 +25,14 @@ %% Continuation data is whatever data returned by any of the parse %% functions. This is used to continue where we left off the next time %% the user calls parse/2. --type continuation_data() :: any(). --type parser_state() :: status_continue | bulk_continue | multibulk_continue | error_continue. +-type continuation_data() :: + start | + {status_continue, Acc :: binary()} | + {error_continue, Acc :: binary()} | + {bulk_size, Acc :: binary()} | + {multibulk_size, Acc :: binary()} | + {bulk_continue, BytesLeft :: integer(), Acc :: binary()} | + {multibulk_continue, NumLeft :: integer(), Acc :: list()}. %% Internal types -ifdef(OTP_RELEASE). % OTP >= 21 @@ -42,8 +48,7 @@ %% Internal parser state. Is returned from parse/2 and must be %% included on the next calls to parse/2. -record(pstate, { - state = undefined :: parser_state() | undefined, - continuation_data :: continuation_data() | undefined + states = [] :: [continuation_data()] }). -define(NL, "\r\n"). diff --git a/src/eredis_parser.erl b/src/eredis_parser.erl index 6e9dfe6..4aec924 100644 --- a/src/eredis_parser.erl +++ b/src/eredis_parser.erl @@ -26,10 +26,6 @@ -export([init/0, parse/2]). -%% Exported for testing --export([parse_bulk/1, parse_bulk/2, - parse_multibulk/1, parse_multibulk/2, buffer_create/0, buffer_create/1]). - %% %% API %% @@ -66,247 +62,134 @@ init() -> %% Parser in initial state, the data we receive will be the beginning %% of a response -parse(#pstate{state = undefined} = State, NewData) -> +parse(#pstate{states = []}, NewData) -> + return(do_parse(start, NewData), []); +parse(#pstate{states = [State | States]}, NewData) -> + return(do_parse(State, NewData), States). + +%% Combines the result of do_parse/2 with the nested states of parse/2. +return({Tag, Value, <<>>}, []) when Tag =:= ok; Tag =:= error -> + {Tag, Value, #pstate{}}; +return({Tag, Value, RestData}, []) when Tag =:= ok; Tag =:= error -> + {Tag, Value, RestData, #pstate{}}; +return({Tag, Value, RestData}, [{multibulk_continue, NumLeft, Acc} | States]) + when Tag =:= ok; Tag =:= error -> + NewStates = [{multibulk_continue, NumLeft - 1, [Value | Acc]} | States], + parse(#pstate{states = NewStates}, RestData); +return({continue, Continue}, States) -> + {continue, #pstate{states = [Continue | States]}}; +return({nested, State, Data}, States) -> + %% We're in a multibulk and need to parse a new element + parse(#pstate{states = [start, State | States]}, Data); +return({error, _Reason} = ParseError, _States) -> + ParseError. + +%% Parses a value. State is not nested here. +-spec do_parse(continuation_data(), NewData :: binary()) -> + {ok, Value :: any(), RestData :: binary()} | + {error, Message :: binary(), RestData :: binary()} | + {continue, continuation_data()} | + {nested, continuation_data(), RestData :: binary()} | + {error, unknown_response}. +do_parse(start, <>) -> %% Look at the first byte to get the type of reply - case NewData of - %% Status - <<$+, Data/binary>> -> - return_result(parse_simple(Data), State, status_continue); + case Type of + %% Status (AKA simple string) + $+ -> + do_parse({status_continue, <<>>}, Data); %% Error - <<$-, Data/binary>> -> - return_error(parse_simple(Data), State, error_continue); + $- -> + do_parse({error_continue, <<>>}, Data); - %% Integer reply - <<$:, Data/binary>> -> - return_result(parse_simple(Data), State, status_continue); + %% Integer reply (returned as binary) + $: -> + do_parse({status_continue, <<>>}, Data); - %% Multibulk - <<$*, _Rest/binary>> -> - return_result(parse_multibulk(NewData), State, multibulk_continue); + %% Multibulk (array) + $* -> + do_parse({multibulk_size, <<>>}, Data); - %% Bulk - <<$$, _Rest/binary>> -> - return_result(parse_bulk(NewData), State, bulk_continue); + %% Bulk (string) + $$ -> + do_parse({bulk_size, <<>>}, Data); _ -> - %% TODO: Handle the case where we start parsing a new - %% response, but cannot make any sense of it {error, unknown_response} end; - -%% The following clauses all match on different continuation states - -parse(#pstate{state = bulk_continue, - continuation_data = ContinuationData} = State, NewData) -> - return_result(parse_bulk(ContinuationData, NewData), State, bulk_continue); - -parse(#pstate{state = multibulk_continue, - continuation_data = ContinuationData} = State, NewData) -> - return_result(parse_multibulk(ContinuationData, NewData), State, multibulk_continue); - -parse(#pstate{state = status_continue, - continuation_data = ContinuationData} = State, NewData) -> - return_result(parse_simple(ContinuationData, NewData), State, status_continue); - -parse(#pstate{state = error_continue, - continuation_data = ContinuationData} = State, NewData) -> - return_error(parse_simple(ContinuationData, NewData), State, error_continue). - -%% -%% MULTIBULK -%% - -parse_multibulk(Data) when is_binary(Data) -> parse_multibulk(buffer_create(Data)); - -parse_multibulk(Buffer) -> - case get_newline_pos(Buffer) of - undefined -> - {continue, {incomplete_size, Buffer}}; - NewlinePos -> - OffsetNewlinePos = NewlinePos - 1, - <<$*, Size:OffsetNewlinePos/binary, ?NL, Bulk/binary>> = buffer_to_binary(Buffer), - IntSize = list_to_integer(binary_to_list(Size)), - - do_parse_multibulk(IntSize, buffer_create(Bulk)) - end. - -%% Size of multibulk was incomplete, try again -parse_multibulk({incomplete_size, Buffer}, NewData0) -> - NewBuffer = buffer_append(Buffer, NewData0), - parse_multibulk(NewBuffer); - -%% Ran out of data inside do_parse_multibulk in parse_bulk, must -%% continue traversing the bulks -parse_multibulk({in_parsing_bulks, Count, Buffer, Acc}, - NewData0) -> - NewBuffer = buffer_append(Buffer, NewData0), - - %% Continue where we left off - do_parse_multibulk(Count, NewBuffer, Acc). - -%% @doc: Parses the given number of bulks from Data. If Data does not -%% contain enough bulks, {continue, ContinuationData} is returned with -%% enough information to start parsing with the correct count and -%% accumulated data. -do_parse_multibulk(Count, Buffer) -> - do_parse_multibulk(Count, Buffer, []). - -do_parse_multibulk(-1, Buffer, []) -> - {ok, undefined, buffer_to_binary(Buffer)}; -do_parse_multibulk(0, Buffer, Acc) -> - {ok, lists:reverse(Acc), buffer_to_binary(Buffer)}; -do_parse_multibulk(Count, Buffer, Acc) -> - case buffer_size(Buffer) == 0 of - true -> {continue, {in_parsing_bulks, Count, buffer_create(), Acc}}; - false -> - %% Try parsing the first bulk in Data, if it works, we get the - %% extra data back that was not part of the bulk which we can - %% recurse on. If the bulk does not contain enough data, we - %% return with a continuation and enough data to pick up where we - %% left off. In the continuation we will get more data - %% automagically in Data, so parsing the bulk might work. - case parse_bulk(Buffer) of - {ok, Value, Rest} -> - do_parse_multibulk(Count - 1, buffer_create(Rest), [Value | Acc]); - {continue, _} -> - {continue, {in_parsing_bulks, Count, Buffer, Acc}} - end - end. - -%% -%% BULK -%% - -parse_bulk(Data) when is_binary(Data) -> parse_bulk(buffer_create(Data)); - -parse_bulk(Buffer) -> - case buffer_hd(Buffer) of - [$*] -> parse_multibulk(Buffer); - [$+] -> parse_simple(buffer_tl(Buffer)); - [$-] -> parse_simple(buffer_tl(Buffer)); - [$:] -> parse_simple(buffer_tl(Buffer)); - [$$] -> do_parse_bulk(Buffer) - end. - -%% Bulk, at beginning of response -do_parse_bulk(Buffer) -> +do_parse({StateTag, Acc}, Data) when StateTag =:= status_continue; + StateTag =:= error_continue -> + case split_by_newline(Acc, Data) of + nomatch -> + {continue, {StateTag, <>}}; + {Value, RestData} -> + Tag = case StateTag of + status_continue -> ok; + error_continue -> error + end, + {Tag, Value, RestData} + end; +do_parse({StateTag, Acc}, Data) when StateTag =:= bulk_size; + StateTag =:= multibulk_size -> %% Find the position of the first terminator, everything up until %% this point contains the size specifier. If we cannot find it, %% we received a partial response and need more data - case get_newline_pos(Buffer) of - undefined -> - {continue, {incomplete_size, Buffer}}; - NewlinePos -> - OffsetNewlinePos = NewlinePos - 1, % Take into account the first $ - <<$$, Size:OffsetNewlinePos/binary, Bulk/binary>> = buffer_to_binary(Buffer), - IntSize = list_to_integer(binary_to_list(Size)), - - if - %% Nil response from redis - IntSize =:= -1 -> - <> = Bulk, - {ok, undefined, Rest}; - %% We have enough data for the entire bulk - size(Bulk) - (size(<>) * 2) >= IntSize -> - <> = Bulk, - {ok, Value, Rest}; - true -> - %% Need more data, so we send the bulk without the - %% size specifier to our future self - {continue, {IntSize, buffer_create(Bulk)}} - end - end. - -%% Bulk, continuation from partial bulk size -parse_bulk({incomplete_size, Buffer}, NewData0) -> - NewBuffer = buffer_append(Buffer, NewData0), - parse_bulk(NewBuffer); - -%% Bulk, continuation from partial bulk value -parse_bulk({IntSize, Buffer0}, Data) -> - Buffer = buffer_append(Buffer0, Data), - - case buffer_size(Buffer) - (size(<>) * 2) >= IntSize of - true -> - <> = buffer_to_binary(Buffer), - {ok, Value, Rest}; - false -> - {continue, {IntSize, Buffer}} - end. - - -%% -%% SIMPLE REPLIES -%% -%% Handles replies on the following format: -%% TData\r\n -%% Where T is a type byte, like '+', '-', ':'. Data is terminated by \r\n - -%% @doc: Parse simple replies. Data must not contain type -%% identifier. Type must be handled by the caller. -parse_simple(Data) when is_binary(Data) -> parse_simple(buffer_create(Data)); - -parse_simple(Buffer) -> - case get_newline_pos(Buffer) of - undefined -> - {continue, {incomplete_simple, Buffer}}; - NewlinePos -> - <> = buffer_to_binary(Buffer), - {ok, Value, Rest} - end. - -parse_simple({incomplete_simple, Buffer}, NewData0) -> - NewBuffer = buffer_append(Buffer, NewData0), - parse_simple(NewBuffer). - -%% -%% INTERNAL HELPERS -%% -get_newline_pos({B, _}) -> - case re:run(B, ?NL) of - {match, [{Pos, _}]} -> Pos; - nomatch -> undefined - end. - -buffer_create() -> - {[], 0}. - -buffer_create(Data) -> - {[Data], byte_size(Data)}. - -buffer_append({List, Size}, Binary) -> - NewList = case List of - [] -> [Binary]; - [Head | Tail] -> [Head, Tail, Binary] - end, - {NewList, Size + byte_size(Binary)}. - -buffer_hd({[<> | _], _}) -> [Char]; -buffer_hd({[], _}) -> []. - -buffer_tl({[<<_, RestBin/binary>> | Rest], Size}) -> {[RestBin | Rest], Size - 1}. - -buffer_to_binary({List, _}) -> iolist_to_binary(List). - -buffer_size({_, Size}) -> Size. - -%% @doc: Helper for handling the result of parsing. Will update the -%% parser state with the continuation of given name if necessary. -return_result({ok, Value, <<>>}, _State, _StateName) -> - {ok, Value, init()}; -return_result({ok, Value, Rest}, _State, _StateName) -> - {ok, Value, Rest, init()}; -return_result({continue, ContinuationData}, State, StateName) -> - {continue, State#pstate{state = StateName, continuation_data = ContinuationData}}. - -%% @doc: Helper for returning an error. Uses return_result/3 and just transforms the {ok, ...} tuple into an error tuple -return_error(Result, State, StateName) -> - case return_result(Result, State, StateName) of - {ok, Value, ParserState} -> - {error, Value, ParserState}; - {ok, Value, Rest, ParserState} -> - {error, Value, Rest, ParserState}; - Res -> - Res + case split_by_newline(Acc, Data) of + nomatch -> + %% Incomplete size + {continue, {StateTag, <>}}; + {Size, RestData} -> + IntSize = binary_to_integer(Size), + NextState = case StateTag of + bulk_size -> {bulk_continue, IntSize, <<>>}; + multibulk_size -> {multibulk_continue, IntSize, []} + end, + do_parse(NextState, RestData) + end; +do_parse({bulk_continue, -1, <<>>}, Data) -> + %% Nil (AKA null) string + {ok, undefined, Data}; +do_parse({bulk_continue, -1, Acc}, <<"\n", RestData/binary>>) when byte_size(Acc) > 0 -> + %% It's only half of the "\r\n" we're waiting for (unlikely case) + BulkSize = byte_size(Acc) - 1, + <> = Acc, + {ok, Bulk, RestData}; +do_parse({bulk_continue, RemainingSize, Acc}, Data) + when byte_size(Data) >= RemainingSize + length(?NL) -> + %% We have enough data for the entire bulk + <> = Data, + Bulk = <>, + {ok, Bulk, RestData}; +do_parse({bulk_continue, RemainingSize, Acc}, Data) -> + NewRemainingSize = RemainingSize - byte_size(Data), + NewAcc = <>, + {continue, {bulk_continue, NewRemainingSize, NewAcc}}; +do_parse({multibulk_continue, -1, []}, Data) -> + %% Nil (AKA null) array + {ok, undefined, Data}; +do_parse({multibulk_continue, 0, Acc}, Data) -> + {ok, lists:reverse(Acc), Data}; +do_parse({multibulk_continue, _RemainingItems, _Acc} = State, <<>>) -> + {continue, State}; +do_parse({multibulk_continue, _RemainingItems, _Acc} = State, Data) -> + {nested, State, Data}. + +%% Concat two binaries and then split by "\r\n", but without actually +%% concatenating, thus avoiding creating a large binary which becomes garbage. +%% +%% Pre-condition: Acc does not contain "\r\n". +split_by_newline(Acc, <<"\n", Rest/binary>>) + when binary_part(Acc, byte_size(Acc), -1) =:= <<"\r">> -> + %% Special case where the "\r\n" sequence is split between Acc and Data + FirstLine = binary_part(Acc, 0, byte_size(Acc) - 1), + {FirstLine, Rest}; +split_by_newline(Acc, Data) -> + %% There's no "\r\n" in Acc so we can search only in Data. + case binary:match(Data, <<"\r\n">>) of + nomatch -> + nomatch; + {NewlinePos, 2} -> + <> = Data, + FirstLine = <>, + {FirstLine, Rest} end. diff --git a/test/eredis_parser_tests.erl b/test/eredis_parser_tests.erl index 5391475..cb9157d 100644 --- a/test/eredis_parser_tests.erl +++ b/test/eredis_parser_tests.erl @@ -9,8 +9,7 @@ -include("eredis.hrl"). -include_lib("eunit/include/eunit.hrl"). --import(eredis_parser, [parse/2, init/0, parse_bulk/1, parse_bulk/2, - parse_multibulk/1, parse_multibulk/2, buffer_create/0, buffer_create/1]). +-import(eredis_parser, [parse/2, init/0]). %% Unknown types unknown_crap_test() -> @@ -34,14 +33,7 @@ status_chunked_test() -> B3 = <<"\r\n">>, State1 = init(), - ?assertEqual({continue, #pstate{state = status_continue, - continuation_data = {incomplete_simple, buffer_create(<<"">>)}}}, - parse(State1, B1)), {continue, State2} = parse(State1, B1), - - ?assertEqual({continue, #pstate{state = status_continue, - continuation_data = {incomplete_simple, {[<<"">>, [], <<"OK">>], 2}}}}, - parse(State2, B2)), {continue, State3} = parse(State2, B2), ?assertEqual({ok, <<"OK">>, init()}, parse(State3, B3)). @@ -61,10 +53,6 @@ error_chunked_test() -> State1 = init(), {continue, State2} = parse(State1, B1), - Buffer = buffer_create(<<"ERR">>), - ?assertEqual(#pstate{state = error_continue, continuation_data = {incomplete_simple, Buffer}}, - State2), - ?assertEqual({error, <<"ERR wrong number of arguments for 'get' command">>, init()}, parse(State2, B2)). @@ -85,10 +73,6 @@ integer_chunked_test() -> State1 = init(), {continue, State2} = parse(State1, B1), - ?assertEqual(#pstate{state = status_continue, - continuation_data = {incomplete_simple, buffer_create(<<"2">>)}}, - State2), - ?assertEqual({ok, <<"25">>, init()}, parse(State2, B2)). %% Bulk test @@ -107,10 +91,6 @@ bulk_chunked_test() -> B2 = <<"bar\r\n">>, {continue, State2} = parse(State1, B1), - Buffer = buffer_create(<<"\r\n">>), - ?assertEqual(#pstate{state = bulk_continue, continuation_data = {3, Buffer}}, - State2), - ?assertEqual({ok, <<"bar">>, init()}, parse(State2, B2)). bulk_multichunked_test() -> @@ -123,30 +103,23 @@ bulk_multichunked_test() -> B6 = <<"\n">>, %% 13 bytes {continue, State2} = parse(State1, B1), - Buffer1 = buffer_create(<<"$1">>), - ?assertEqual(#pstate{state = bulk_continue, - continuation_data = {incomplete_size, Buffer1}}, + ?assertEqual(#pstate{states = [{bulk_size, <<"1">>}]}, State2), {continue, State3} = parse(State2, B2), - Buffer2 = buffer_create(<<"\r\n">>), - ?assertEqual(#pstate{state = bulk_continue, - continuation_data = {13, Buffer2}}, + ?assertEqual(#pstate{states = [{bulk_continue, 13, <<>>}]}, State3), {continue, State4} = parse(State3, B3), - ?assertEqual(#pstate{state = bulk_continue, - continuation_data = {13, {[<<"\r\n">>, [], <<"foobar">>], 8}}}, + ?assertEqual(#pstate{states = [{bulk_continue, 7, <<"foobar">>}]}, State4), {continue, State5} = parse(State4, B4), - ?assertEqual(#pstate{state = bulk_continue, - continuation_data = {13, {[<<"\r\n">>, [[], <<"foobar">>], <<"bazquux">>], 15}}}, + ?assertEqual(#pstate{states = [{bulk_continue, 0, <<"foobarbazquux">>}]}, State5), {continue, State6} = parse(State5, B5), - ?assertEqual(#pstate{state = bulk_continue, - continuation_data = {13, {[<<"\r\n">>, [[[], <<"foobar">>], <<"bazquux">>], <<"\r">>], 16}}}, + ?assertEqual(#pstate{states = [{bulk_continue, -1, <<"foobarbazquux\r">>}]}, State6), ?assertEqual({ok, <<"foobarbazquux">>, init()}, parse(State6, B6)). @@ -161,10 +134,6 @@ bulk_empty_chunked_test() -> State1 = init(), {continue, State2} = parse(State1, B1), - Buffer = buffer_create(<<"\r\n\r">>), - ?assertEqual(#pstate{state = bulk_continue, continuation_data = {0, Buffer}}, - State2), - ?assertEqual({ok, <<"">>, init()}, parse(State2, B2)). bulk_empty_and_rest_test() -> @@ -179,13 +148,8 @@ bulk_nil_chunked_test() -> State1 = init(), B1 = <<"$-1">>, B2 = <<"\r\n">>, - Buffer = buffer_create(<<"$-1">>), - ?assertEqual({continue, #pstate{state = bulk_continue, - continuation_data = {incomplete_size, Buffer}}}, - parse(State1, B1)), {continue, State2} = parse(State1, B1), - ?assertEqual({ok, undefined, init()}, parse(State2, B2)). bulk_nil_and_rest_test() -> @@ -195,55 +159,15 @@ bulk_nil_and_rest_test() -> %% parse_bulk function tests parse_bulk_test() -> B = <<"$3\r\nbar\r\n">>, - ?_assertEqual({ok, <<"bar">>, <<>>}, parse_bulk(B)). - -parse_bulk_split_test() -> - B1 = <<"$3\r\n">>, - B2 = <<"bar\r\n">>, - - Buffer = buffer_create(<<"\r\n">>), - ?assertEqual({continue, {3, Buffer}}, parse_bulk(B1)), - {continue, Res} = parse_bulk(B1), - ?assertEqual({ok, <<"bar">>, <<>>}, parse_bulk(Res, B2)). - -parse_bulk_very_split_test() -> - B1 = <<"$1">>, - B2 = <<"3\r\n">>, - B3 = <<"foobarbazquux\r\n">>, %% 13 bytes - - Buffer1 = buffer_create(<<"$1">>), - ?assertEqual({continue, {incomplete_size, Buffer1}}, parse_bulk(B1)), - {continue, ContinuationData1} = parse_bulk(B1), - - Buffer2 = buffer_create(<<"\r\n">>), - ?assertEqual({continue, {13, Buffer2}}, parse_bulk(ContinuationData1, B2)), - {continue, ContinuationData2} = parse_bulk(ContinuationData1, B2), - - ?assertEqual({ok, <<"foobarbazquux">>, <<>>}, parse_bulk(ContinuationData2, B3)). - -parse_bulk_split_on_newline_test() -> - B1 = <<"$13\r\nfoobarbazquux">>, - B2 = <<"\r\n">>, %% 13 bytes - - Buffer = buffer_create(<<"\r\nfoobarbazquux">>), - ?assertEqual({continue, {13, Buffer}}, parse_bulk(B1)), - {continue, ContinuationData1} = parse_bulk(B1), - ?assertEqual({ok, <<"foobarbazquux">>, <<>>}, parse_bulk(ContinuationData1, B2)). - -parse_bulk_too_much_data_test() -> - B = <<"$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n">>, - ?assertEqual({ok, <<"1">>, <<"$1\r\n2\r\n$1\r\n3\r\n">>}, parse_bulk(B)). + ?_assertEqual({ok, <<"bar">>, <<>>}, parse(init(), B)). parse_bulk_too_much_data_in_continuation_test() -> B1 = <<"$1\r\n">>, B2 = <<"1\r\n$1\r\n2\r\n$1\r\n3\r\n">>, - Buffer = buffer_create(<<"\r\n">>), - ?assertEqual({continue, {1, Buffer}}, parse_bulk(B1)), - {continue, ContinuationData1} = parse_bulk(B1), - - ?assertEqual({ok, <<"1">>, <<"$1\r\n2\r\n$1\r\n3\r\n">>}, - parse_bulk(ContinuationData1, B2)). + {continue, ContinuationData1} = parse(init(), B1), + ?assertEqual({ok, <<"1">>, <<"$1\r\n2\r\n$1\r\n3\r\n">>, init()}, + parse(ContinuationData1, B2)). %% Multibulk test / RESP Arrays multibulk_empty_test() -> @@ -252,50 +176,32 @@ multibulk_empty_test() -> ?assertEqual({ok, [], #pstate{}}, parse(init(), B)). multibulk_test() -> - %% [{1, 1}, {2, 2}, {3, 3}] + %% ["1", "2", "3"] B = <<"*3\r\n$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n">>, ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], #pstate{}}, parse(init(), B)). multibulk_one_byte_parse_test() -> - %% [{1, 1}, {2, 2}, {3, 3}] + %% ["1", "2", "3"] B = <<"*3\r\n$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n">>, ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], #pstate{}}, one_byte_parse(B)). multibulk_split_parse_test() -> - %% [{1, 1}, {2, 2}, {3, 3}] + %% ["1", "2", "3"] B1 = <<"*3\r\n$1\r\n1\r\n$1">>, B2 = <<"\r\n2\r\n$1\r\n3\r\n">>, State1 = init(), - - Buffer = buffer_create(<<"$1">>), - ?assertEqual({continue, - #pstate{state = multibulk_continue, - continuation_data = - {in_parsing_bulks, 2, Buffer, [<<"1">>]}}}, - parse(State1, B1)), - {continue, State2} = parse(State1, B1), - ?assertMatch({ok, [<<"1">>, <<"2">>, <<"3">>], _}, parse(State2, B2)). multibulk_split_parse_with_rest_test() -> - %% [{1, 1}, {2, 2}, {3, 3}] + %% ["1", "2", "3"] B1 = <<"*3\r\n$1\r\n1\r\n$1">>, B2 = <<"\r\n2\r\n$1\r\n3\r\nDATA">>, State1 = init(), - - Buffer = buffer_create(<<"$1">>), - ?assertEqual({continue, - #pstate{state = multibulk_continue, - continuation_data = - {in_parsing_bulks, 2, Buffer, [<<"1">>]}}}, - parse(State1, B1)), - {continue, State2} = parse(State1, B1), - ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], <<"DATA">>, #pstate{}}, parse(State2, B2)). @@ -334,7 +240,7 @@ nested_multibulk_parse_test() -> parse(init(), B)). nested_multibulk_one_byte_parse_test() -> - %% [[1, 2], [3, 4]] + %% [["1", "2"], ["3", "4"]] B = <<"*2\r\n*2\r\n$1\r\n1\r\n$1\r\n2\r\n*2\r\n$1\r\n3\r\n$1\r\n4\r\n">>, ?assertEqual({ok, [[<<"1">>, <<"2">>], [<<"3">>, <<"4">>]], #pstate{}}, one_byte_parse(B)). @@ -358,43 +264,37 @@ multibulk_big_chunks_test() -> %% Real-world example, MGET 1..200 B1 = <<"*200\r\n$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n$1\r\n4\r\n$1\r\n5\r\n$1\r\n6\r\n$1\r\n7\r\n$1\r\n8\r\n$1\r\n9\r\n$2\r\n10\r\n$2\r\n11\r\n$2\r\n12\r\n$2\r\n13\r\n$2\r\n14\r\n$2\r\n15\r\n$2\r\n16\r\n$2\r\n17\r\n$2\r\n18\r\n$2\r\n19\r\n$2\r\n20\r\n$2\r\n21\r\n$2\r\n22\r\n$2\r\n23\r\n$2\r\n24\r\n$2\r\n25\r\n$2\r\n26\r\n$2\r\n27\r\n$2\r\n28\r\n$2\r\n29\r\n$2\r\n30\r\n$2\r\n31\r\n$2\r\n32\r\n$2\r\n33\r\n$2\r\n34\r\n$2\r\n35\r\n$2\r\n36\r\n$2\r\n37\r\n$2\r\n38\r\n$2\r\n39\r\n$2\r\n40\r\n$2\r\n41\r\n$2\r\n42\r\n$2\r\n43\r\n$2\r\n44\r\n$2\r\n45\r\n$2\r\n46\r\n$2\r\n47\r\n$2\r\n48\r\n$2\r\n49\r\n$2\r\n50\r\n$2\r\n51\r\n$2\r\n52\r\n$2\r\n53\r\n$2\r\n54\r\n$2\r\n55\r\n$2\r\n56\r\n$2\r\n57\r\n$2\r\n58\r\n$2\r\n59\r\n$2\r\n60\r\n$2\r\n61\r\n$2\r\n62\r\n$2\r\n63\r\n$2\r\n64\r\n$2\r\n65\r\n$2\r\n66\r\n$2\r\n67\r\n$2\r\n68\r\n$2\r\n69\r\n$2\r\n70\r\n$2\r\n71\r\n$2\r\n72\r\n$2\r\n73\r\n$2\r\n74\r\n$2\r\n75\r\n$2\r\n76\r\n$2\r\n77\r\n$2\r\n78\r\n$2\r\n79\r\n$2\r\n80\r\n$2\r\n81\r\n$2\r\n82\r\n$2\r\n83\r\n$2\r\n84\r\n$2\r\n85\r\n$2\r\n86\r\n$2\r\n87\r\n$2\r\n88\r\n$2\r\n89\r\n$2\r\n90\r\n$2\r\n91\r\n$2\r\n92\r\n$2\r\n93\r\n$2\r\n94\r\n$2\r\n95\r\n$2\r\n96\r\n$2\r\n97\r\n$2\r\n98\r\n$2\r\n99\r\n$3\r\n100\r\n$3\r\n101\r\n$3\r\n102\r\n$3\r\n103\r\n$3\r\n104\r\n$3\r\n105\r\n$3\r\n106\r\n$3\r\n107\r\n$3\r\n108\r\n$3\r\n109\r\n$3\r\n110\r\n$3\r\n111\r\n$3\r\n112\r\n$3\r\n113\r\n$3\r\n114\r\n$3\r\n115\r\n$3\r\n116\r\n$3\r\n117\r\n$3\r\n118\r\n$3\r\n119\r\n$3\r\n120\r\n$3\r\n121\r\n$3\r\n122\r\n$3\r\n123\r\n$3\r\n124\r\n$3\r\n125\r\n$3\r\n126\r\n$3\r\n127\r\n$3\r\n128\r\n$3\r\n129\r\n$3\r\n130\r\n$3\r\n131\r\n$3\r\n132\r\n$3\r\n133\r\n$3\r\n134\r\n$3\r\n135\r\n$3\r\n136\r\n$3\r\n137\r\n$3\r\n138\r\n$3\r\n139\r\n$3\r\n140\r\n$3\r\n141\r\n$3\r\n142\r\n$3\r\n143\r\n$3\r\n144\r\n$3\r\n145\r\n$3\r\n146\r\n$3\r\n147\r\n$3\r\n148\r\n$3\r\n149\r\n$3\r\n150\r\n$3\r\n151\r\n$3\r\n152\r\n$3\r\n153\r\n$3\r\n154\r\n$3\r\n155\r\n$3\r\n156\r\n$3\r\n157\r\n$3\r\n158\r\n$3\r\n159\r\n$3\r\n160\r\n$3\r\n161\r\n$3\r\n162\r\n$3\r\n163\r\n$3\r\n164\r\n$3\r\n165\r\n$3\r\n166\r\n$3\r\n167\r\n$3\r\n168\r\n$3\r\n169\r\n$3\r\n170\r\n$3\r\n171\r\n$3\r\n172\r\n$3\r\n173\r\n$3\r\n1">>, B2 = <<"74\r\n$3\r\n175\r\n$3\r\n176\r\n$3\r\n177\r\n$3\r\n178\r\n$3\r\n179\r\n$3\r\n180\r\n$3\r\n181\r\n$3\r\n182\r\n$3\r\n183\r\n$3\r\n184\r\n$3\r\n185\r\n$3\r\n186\r\n$3\r\n187\r\n$3\r\n188\r\n$3\r\n189\r\n$3\r\n190\r\n$3\r\n191\r\n$3\r\n192\r\n$3\r\n193\r\n$3\r\n194\r\n$3\r\n195\r\n$3\r\n196\r\n$3\r\n197\r\n$3\r\n198\r\n$3\r\n199\r\n$3\r\n200\r\n">>, - ExpectedValues = [list_to_binary(integer_to_list(N)) || N <- lists:seq(1, 200)], + ExpectedValues = [integer_to_binary(N) || N <- lists:seq(1, 200)], State1 = init(), - ?assertMatch({continue, - #pstate{state = multibulk_continue, - continuation_data = {in_parsing_bulks, 27, _, _}}}, - parse(State1, B1)), {continue, State2} = parse(State1, B1), - - ?assertMatch({ok, ExpectedValues, #pstate{state = undefined, - continuation_data = undefined}}, + ?assertEqual({ok, ExpectedValues, #pstate{}}, parse(State2, B2)). %% parse_multibulk function tests parse_multibulk_test() -> - %% [{1, 1}, {2, 2}, {3, 3}] + %% ["1", "2", "3"] B = <<"*3\r\n$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n">>, - ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], <<>>}, parse_multibulk(B)). + ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], init()}, parse(init(), B)). parse_multibulk_nested_test() -> - %% [[1, 2], [3, 4]] + %% [["1", "2"], ["3", "4"]] B = <<"*2\r\n*2\r\n$1\r\n1\r\n$1\r\n2\r\n*2\r\n$1\r\n3\r\n$1\r\n4\r\n">>, - ?assertEqual({ok, [[<<"1">>, <<"2">>], [<<"3">>, <<"4">>]], <<>>}, - parse_multibulk(B)). + ?assertEqual({ok, [[<<"1">>, <<"2">>], [<<"3">>, <<"4">>]], init()}, + parse(init(), B)). parse_multibulk_nil_test() -> B = <<"*-1\r\n">>, - ?assertEqual({ok, undefined, <<>>}, parse_multibulk(B)). + ?assertEqual({ok, undefined, init()}, parse(init(), B)). parse_multibulk_split_test() -> %% Split into 2 parts: <<"*3\r\n$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n">> B1 = <<"*3\r\n$1\r\n1\r\n$1">>, B2 = <<"\r\n2\r\n$1\r\n3\r\n">>, - {continue, ContinuationData1} = parse_multibulk(B1), - Result = parse_multibulk(ContinuationData1, B2), - ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], <<>>}, Result). + {continue, ContinuationData1} = parse(init(), B1), + Result = parse(ContinuationData1, B2), + ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], init()}, Result). parse_multibulk_very_split_test() -> %% Split into 4 parts: <<"*3\r\n$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n">> @@ -403,38 +303,56 @@ parse_multibulk_very_split_test() -> B3 = <<"\n1\r\n$1\r\n2\r\n$1">>, B4 = <<"\r\n3\r\n">>, - Buffer = buffer_create(<<"*">>), - ?assertEqual({continue, {incomplete_size, Buffer}}, parse_multibulk(B1)), - {continue, ContinuationData1} = parse_multibulk(B1), - {continue, ContinuationData2} = parse_multibulk(ContinuationData1, B2), - {continue, ContinuationData3} = parse_multibulk(ContinuationData2, B3), - - Result = parse_multibulk(ContinuationData3, B4), - ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], <<>>}, Result). + {continue, ContinuationData1} = parse(init(), B1), + {continue, ContinuationData2} = parse(ContinuationData1, B2), + {continue, ContinuationData3} = parse(ContinuationData2, B3), + Result = parse(ContinuationData3, B4), + ?assertEqual({ok, [<<"1">>, <<"2">>, <<"3">>], init()}, Result). parse_multibulk_newline_split_test() -> %% Split into 4 parts: <<"*3\r\n$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n">> B1 = <<"*2\r\n$1\r\n1">>, B2 = <<"\r\n$1\r\n2\r\n">>, - Buffer = buffer_create(<<"$1\r\n1">>), - ?assertEqual({continue, {in_parsing_bulks, 2, Buffer, []}}, - parse_multibulk(B1)), - {continue, ContinuationData1} = parse_multibulk(B1), - - ?assertEqual({ok, [<<"1">>, <<"2">>], <<>>}, parse_multibulk(ContinuationData1, B2)). + {continue, ContinuationData1} = parse(init(), B1), + ?assertEqual({ok, [<<"1">>, <<"2">>], init()}, parse(ContinuationData1, B2)). parse_multibulk_chunk_test() -> B1 = <<"*500\r\n$1\r\n1\r\n$1\r\n2\r\n$1\r\n3\r\n$1\r\n4\r\n$1\r\n5\r\n$1\r\n6\r\n$1\r\n7\r\n$1\r\n8\r\n$1\r\n9\r\n$2\r\n10\r\n$2\r\n11\r\n$2\r\n12\r\n$2\r\n13\r\n$2\r\n14\r\n$2\r\n15\r\n$2\r\n16\r\n$2\r\n17\r\n$2\r\n18\r\n$2\r\n19\r\n$2\r\n20\r\n$2\r\n21\r\n$2\r\n22\r\n$2\r\n23\r\n$2\r\n24\r\n$2\r\n25\r\n$2\r\n26\r\n$2\r\n27\r\n$2\r\n28\r\n$2\r\n29\r\n$2\r\n30\r\n$2\r\n31\r\n$2\r\n32\r\n$2\r\n33\r\n$2\r\n34\r\n$2\r\n35\r\n$2\r\n36\r\n$2\r\n37\r\n$2\r\n38\r\n$2\r\n39\r\n$2\r\n40\r\n$2\r\n41\r\n$2\r\n42\r\n$2\r\n43\r\n$2\r\n44\r\n$2\r\n45\r\n$2\r\n46\r\n$2\r\n47\r\n$2\r\n48\r\n$2\r\n49\r\n$2\r\n50\r\n$2\r\n51\r\n$2\r\n52\r\n$2\r\n53\r\n$2\r\n54\r\n$2\r\n55\r\n$2\r\n56\r\n$2\r\n57\r\n$2\r\n58\r\n$2\r\n59\r\n$2\r\n60\r\n$2\r\n61\r\n$2\r\n62\r\n$2\r\n63\r\n$2\r\n64\r\n$2\r\n65\r\n$2\r\n66\r\n$2\r\n67\r\n$2\r\n68\r\n$2\r\n69\r\n$2\r\n70\r\n$2\r\n71\r\n$2\r\n72\r\n$2\r\n73\r\n$2\r\n74\r\n$2\r\n75\r\n$2\r\n76\r\n$2\r\n77\r\n$2\r\n78\r\n$2\r\n79\r\n$2\r\n80\r\n$2\r\n81\r\n$2\r\n82\r\n$2\r\n83\r\n$2\r\n84\r\n$2\r\n85\r\n$2\r\n86\r\n$2\r\n87\r\n$2\r\n88\r\n$2\r\n89\r\n$2\r\n90\r\n$2\r\n91\r\n$2\r\n92\r\n$2\r\n93\r\n$2\r\n94\r\n$2\r\n95\r\n$2\r\n96\r\n$2\r\n97\r\n$2\r\n98\r\n$2\r\n99\r\n$3\r\n100\r\n$3\r\n101\r\n$3\r\n102\r\n$3\r\n103\r\n$3\r\n104\r\n$3\r\n105\r\n$3\r\n106\r\n$3\r\n107\r\n$3\r\n108\r\n$3\r\n109\r\n$3\r\n110\r\n$3\r\n111\r\n$3\r\n112\r\n$3\r\n113\r\n$3\r\n114\r\n$3\r\n115\r\n$3\r\n116\r\n$3\r\n117\r\n$3\r\n118\r\n$3\r\n119\r\n$3\r\n120\r\n$3\r\n121\r\n$3\r\n122\r\n$3\r\n123\r\n$3\r\n124\r\n$3\r\n125\r\n$3\r\n126\r\n$3\r\n127\r\n$3\r\n128\r\n$3\r\n129\r\n$3\r\n130\r\n$3\r\n131\r\n$3\r\n132\r\n$3\r\n133\r\n$3\r\n134\r\n$3\r\n135\r\n$3\r\n136\r\n$3\r\n137\r\n$3\r\n138\r\n$3\r\n139\r\n$3\r\n140\r\n$3\r\n141\r\n$3\r\n142\r\n$3\r\n143\r\n$3\r\n144\r\n$3\r\n145\r\n$3\r\n146\r\n$3\r\n147\r\n$3\r\n148\r\n$3\r\n149\r\n$3\r\n150\r\n$3\r\n151\r\n$3\r\n152\r\n$3\r\n153\r\n$3\r\n154\r\n$3\r\n155\r\n$3\r\n156\r\n$3\r\n157\r\n$3\r\n158\r\n$3\r\n159\r\n$3\r\n160\r\n$3\r\n161\r\n$3\r\n162\r\n$3\r\n163\r\n$3\r\n164\r\n$3\r\n165\r\n$3\r\n166\r\n$3\r\n167\r\n$3\r\n168\r\n$3\r\n169\r\n$3\r\n170\r\n$3\r\n171\r\n$3\r\n172\r\n$3\r\n173\r\n$3\r\n1">>, B2 = <<"74\r\n$3\r\n175\r\n$3\r\n176\r\n$3\r\n177\r\n$3\r\n178\r\n$3\r\n179\r\n$3\r\n180\r\n$3\r\n181\r\n$3\r\n182\r\n$3\r\n183\r\n$3\r\n184\r\n$3\r\n185\r\n$3\r\n186\r\n$3\r\n187\r\n$3\r\n188\r\n$3\r\n189\r\n$3\r\n190\r\n$3\r\n191\r\n$3\r\n192\r\n$3\r\n193\r\n$3\r\n194\r\n$3\r\n195\r\n$3\r\n196\r\n$3\r\n197\r\n$3\r\n198\r\n$3\r\n199\r\n$3\r\n200\r\n$3\r\n201\r\n$3\r\n202\r\n$3\r\n203\r\n$3\r\n204\r\n$3\r\n205\r\n$3\r\n206\r\n$3\r\n207\r\n$3\r\n208\r\n$3\r\n209\r\n$3\r\n210\r\n$3\r\n211\r\n$3\r\n212\r\n$3\r\n213\r\n$3\r\n214\r\n$3\r\n215\r\n$3\r\n216\r\n$3\r\n217\r\n$3\r\n218\r\n$3\r\n219\r\n$3\r\n220\r\n$3\r\n221\r\n$3\r\n222\r\n$3\r\n223\r\n$3\r\n224\r\n$3\r\n225\r\n$3\r\n226\r\n$3\r\n227\r\n$3\r\n228\r\n$3\r\n229\r\n$3\r\n230\r\n$3\r\n231\r\n$3\r\n232\r\n$3\r\n233\r\n$3\r\n234\r\n$3\r\n235\r\n$3\r\n236\r\n$3\r\n237\r\n$3\r\n238\r\n$3\r\n239\r\n$3\r\n240\r\n$3\r\n241\r\n$3\r\n242\r\n$3\r\n243\r\n$3\r\n244\r\n$3\r\n245\r\n$3\r\n246\r\n$3\r\n247\r\n$3\r\n248\r\n$3\r\n249\r\n$3\r\n250\r\n$3\r\n251\r\n$3\r\n252\r\n$3\r\n253\r\n$3\r\n254\r\n$3\r\n255\r\n$3\r\n256\r\n$3\r\n257\r\n$3\r\n258\r\n$3\r\n259\r\n$3\r\n260\r\n$3\r\n261\r\n$3\r\n262\r\n$3\r\n263\r\n$3\r\n264\r\n$3\r\n265\r\n$3\r\n266\r\n$3\r\n267\r\n$3\r\n268\r\n$3\r\n269\r\n$3\r\n270\r\n$3\r\n271\r\n$3\r\n272\r\n$3\r\n273\r\n$3\r\n274\r\n$3\r\n275\r\n$3\r\n276\r\n$3\r\n277\r\n$3\r\n278\r\n$3\r\n279\r\n$3\r\n280\r\n$3\r\n281\r\n$3\r\n282\r\n$3\r\n283\r\n$3\r\n284\r\n$3\r\n285\r\n$3\r\n286\r\n$3\r\n287\r\n$3\r\n288\r\n$3\r\n289\r\n$3\r\n290\r\n$3\r\n291\r\n$3\r\n292\r\n$3\r\n293\r\n$3\r\n294\r\n$3\r\n295\r\n$3\r\n296\r\n$3\r\n297\r\n$3\r\n298\r\n$3\r\n299\r\n$3\r\n300\r\n$3\r\n301\r\n$3\r\n302\r\n$3\r\n303\r\n$3\r\n304\r\n$3\r\n305\r\n$3\r\n306\r\n$3\r\n307\r\n$3\r\n308\r\n$3\r\n309\r\n$3\r\n310\r\n$3\r\n311\r\n$3\r\n312\r\n$3\r\n313\r\n$3\r\n314\r\n$3\r\n315\r\n$3\r\n316\r\n$3\r\n317\r\n$3\r\n318\r\n$3\r\n319\r\n$3\r\n320\r\n$3\r\n321\r\n$3\r\n322\r\n$3\r\n323\r\n$3\r\n324\r\n$3\r\n325\r\n$3\r\n326\r\n$3\r\n327\r\n$3\r\n328\r\n$3\r\n329\r\n$3\r\n330\r\n$3\r\n331\r\n$3\r\n332\r\n$3\r\n333\r\n$3\r\n334\r\n$3\r\n335\r\n$3\r\n336">>, B3 = <<"\r\n$3\r\n337\r\n$3\r\n338\r\n$3\r\n339\r\n$3\r\n340\r\n$3\r\n341\r\n$3\r\n342\r\n$3\r\n343\r\n$3\r\n344\r\n$3\r\n345\r\n$3\r\n346\r\n$3\r\n347\r\n$3\r\n348\r\n$3\r\n349\r\n$3\r\n350\r\n$3\r\n351\r\n$3\r\n352\r\n$3\r\n353\r\n$3\r\n354\r\n$3\r\n355\r\n$3\r\n356\r\n$3\r\n357\r\n$3\r\n358\r\n$3\r\n359\r\n$3\r\n360\r\n$3\r\n361\r\n$3\r\n362\r\n$3\r\n363\r\n$3\r\n364\r\n$3\r\n365\r\n$3\r\n366\r\n$3\r\n367\r\n$3\r\n368\r\n$3\r\n369\r\n$3\r\n370\r\n$3\r\n371\r\n$3\r\n372\r\n$3\r\n373\r\n$3\r\n374\r\n$3\r\n375\r\n$3\r\n376\r\n$3\r\n377\r\n$3\r\n378\r\n$3\r\n379\r\n$3\r\n380\r\n$3\r\n381\r\n$3\r\n382\r\n$3\r\n383\r\n$3\r\n384\r\n$3\r\n385\r\n$3\r\n386\r\n$3\r\n387\r\n$3\r\n388\r\n$3\r\n389\r\n$3\r\n390\r\n$3\r\n391\r\n$3\r\n392\r\n$3\r\n393\r\n$3\r\n394\r\n$3\r\n395\r\n$3\r\n396\r\n$3\r\n397\r\n$3\r\n398\r\n$3\r\n399\r\n$3\r\n400\r\n$3\r\n401\r\n$3\r\n402\r\n$3\r\n403\r\n$3\r\n404\r\n$3\r\n405\r\n$3\r\n406\r\n$3\r\n407\r\n$3\r\n408\r\n$3\r\n409\r\n$3\r\n410\r\n$3\r\n411\r\n$3\r\n412\r\n$3\r\n413\r\n$3\r\n414\r\n$3\r\n415\r\n$3\r\n416\r\n$3\r\n417\r\n$3\r\n418\r\n$3\r\n419\r\n$3\r\n420\r\n$3\r\n421\r\n$3\r\n422\r\n$3\r\n423\r\n$3\r\n424\r\n$3\r\n425\r\n$3\r\n426\r\n$3\r\n427\r\n$3\r\n428\r\n$3\r\n429\r\n$3\r\n430\r\n$3\r\n431\r\n$3\r\n432\r\n$3\r\n433\r\n$3\r\n434\r\n$3\r\n435\r\n$3\r\n436\r\n$3\r\n437\r\n$3\r\n438\r\n$3\r\n439\r\n$3\r\n440\r\n$3\r\n441\r\n$3\r\n442\r\n$3\r\n443\r\n$3\r\n444\r\n$3\r\n445\r\n$3\r\n446\r\n$3\r\n447\r\n$3\r\n448\r\n$3\r\n449\r\n$3\r\n450\r\n$3\r\n451\r\n$3\r\n452\r\n$3\r\n453\r\n$3\r\n454\r\n$3\r\n455\r\n$3\r\n456\r\n$3\r\n457\r\n$3\r\n458\r\n$3\r\n459\r\n$3\r\n460\r\n$3\r\n461\r\n$3\r\n462\r\n$3\r\n463\r\n$3\r\n464\r\n$3\r\n465\r\n$3\r\n466\r\n$3\r\n467\r\n$3\r\n468\r\n$3\r\n469\r\n$3\r\n470\r\n$3\r\n471\r\n$3\r\n472\r\n$3\r\n473\r\n$3\r\n474\r\n$3\r\n475\r\n$3\r\n476\r\n$3\r\n477\r\n$3\r\n478\r\n$3\r\n479\r\n$3\r\n480\r\n$3\r\n481\r\n$3\r\n482\r\n$3\r\n483\r\n$3\r\n484\r\n$3\r\n485\r\n$3\r\n486\r\n$3\r\n487\r\n$3\r\n488\r\n$3\r\n489\r\n$3\r\n490\r\n$3\r\n491\r\n$3\r\n492\r\n$3\r\n493\r\n$3\r\n494\r\n$3\r\n495\r\n$3\r\n496\r\n$3\r\n497\r\n$3\r\n498\r\n">>, - - {continue, ContinuationData1} = parse_multibulk(B1), - {continue, ContinuationData2} = parse_multibulk(ContinuationData1, B2), - - EmptyBuffer = buffer_create(), - ?assertMatch({continue, {in_parsing_bulks, 2, EmptyBuffer, _}}, - parse_multibulk(ContinuationData2, B3)). + B4 = <<"$3\r\n499\r\n$3\r\n500\r\n">>, + + {continue, ContinuationData1} = parse(init(), B1), + {continue, ContinuationData2} = parse(ContinuationData1, B2), + {continue, ContinuationData3} = parse(ContinuationData2, B3), + {ok, Value, _State} = parse(ContinuationData3, B4), + ?assertEqual(Value, [integer_to_binary(X) || X <- lists:seq(1, 500)]). + +%% https://github.com/wooga/eredis/issues/127 +%% The problem here is (was) timeout. +parse_multibulk_with_large_bulk_test() -> + A = binary:copy(<<"a">>, 10000000), + B = <<"*2\r\n$1\r\n1\r\n$10000000\r\n", A/binary, "\r\n">>, + ?assertEqual({ok, [<<"1">>, A], init()}, + parse_in_chunks(B, 1460, init())). + +%% Parse a string multiple times, with the data chunk boundaries inside the bulk +%% size part of the protocol. This is a performance thingy. Check the time +%% measurements printed by eunit. +parse_bulks_with_chunk_split_in_size_test() -> + A = binary:copy(<<"a">>, 100000), + Start = <<"$100">>, + End = <<"000\r\n", A/binary, "\r\n">>, + Wrap = <>, + {continue, FinalPstate} = + lists:foldl(fun (_, {continue, Pstate1}) -> + {ok, A, RestData, Pstate2} = parse(Pstate1, Wrap), + parse(Pstate2, RestData) + end, + parse(init(), Start), + lists:seq(1, 10000)), + ?assertEqual({ok, A, init()}, parse(FinalPstate, End)). %% %% Helpers @@ -459,3 +377,17 @@ one_byte_parse(S, <>) -> Other -> Other end. + +%% A wrapper around eredis_parser:parse/2, feeding it with chunks of data. +parse_in_chunks(Data, ChunkSize, ParserState) -> + case Data of + <> -> + case eredis_parser:parse(ParserState, Chunk) of + {continue, NewParserState} -> + parse_in_chunks(Rest, ChunkSize, NewParserState); + Result -> + Result + end; + LastChunk -> + eredis_parser:parse(ParserState, LastChunk) + end.