Skip to content

Added native otp17+ map support #20

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@
JSONX is an Erlang library for efficient JSON decoding and encoding, implemented in Erlang NIFs.
Works with binaries as strings, arrays as lists and only knows how to decode UTF-8 (and ASCII).

Map encoding/decoding supported in Erlang/OTP 17+

JSONX IS VERY FAST!
------------------

Expand Down Expand Up @@ -114,6 +116,10 @@ Examples encoding JSON
%% Object as eep18 propsal
4> jsonx:encode( {[{name, <<"Ivan">>}, {age, 33}, {phones, [3332211, 4443322]}]} ).
<<"{\"name\":\"Ivan\",\"age\":33,\"phones\":[3332211,4443322]}">>

%% Object as Map
5> jsonx:encode( #{name => <<"Ivan">>, age => 33, phones => [3332211, 4443322]} ).
<<"{\"age\":33,\"name\":\"Ivan\",\"phones\":[3332211,4443322]}">>
```

Examples decoding JSON
Expand All @@ -139,6 +145,11 @@ Examples decoding JSON
{struct,[{<<"name">>,<<"Ivan">>},
{<<"age">>,33},
{<<"phones">>,[3332211,4443322]}]}

5> jsonx:decode(<<"{\"name\":\"Ivan\",\"age\":33,\"phones\":[3332211,4443322]}">>, [{format, map}]).
{map,#{<<"age">> => 33,
<<"name">> => <<"Ivan">>,
<<"phones">> => [3332211,4443322]}}
```

Example streaming parse
Expand Down Expand Up @@ -190,6 +201,7 @@ Mapping (JSON -> Erlang)
{"this": "json"} :-> {[{<<"this">>: <<"json">>}]} %% default eep18
{"this": "json"} :-> [{<<"this">>: <<"json">>}] %% optional proplist
{"this": "json"} :-> {struct, [{<<"this">>: <<"json">>}]} %% optional struct
{"this": "json"} :-> {map, #{<<"this">> => <<"json">>}} %% optional map
JSONObject :-> #rec{...} %% decoder must be predefined

Mapping (Erlang -> JSON)
Expand All @@ -202,6 +214,7 @@ Mapping (Erlang -> JSON)
<<"str">> :-> "str"
[1, 2.99] :-> [1, 2.99]
{struct, [{<<"this">>: <<"json">>}]} :-> {"this": "json"}
{map, #{this => <<"json">>}} :-> {"this": "json"}
[{<<"this">>: <<"json">>}] :-> {"this": "json"}
{[{<<"this">>: <<"json">>}]} :-> {"this": "json"}
{json, IOList} :-> `iolist_to_binary(IOList)` %% include with no validation
Expand Down
79 changes: 77 additions & 2 deletions c_src/decoder.c
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include <ctype.h>
#include <errno.h>
#include <assert.h>
#include <stdio.h>
#include "jsonx.h"
#include "jsonx_str.h"

Expand All @@ -17,7 +18,7 @@ typedef struct{
unsigned char *cur;
size_t offset;
ERL_NIF_TERM input;
ERL_NIF_TERM format; //struct, eep18, proplist
ERL_NIF_TERM format; //struct, eep18, proplist, map
ERL_NIF_TERM error;
ERL_NIF_TERM *stack_top;
ERL_NIF_TERM *stack_down;
Expand All @@ -36,6 +37,10 @@ static inline ERL_NIF_TERM parse_true(State* st);
static inline ERL_NIF_TERM parse_false(State* st);
static inline ERL_NIF_TERM parse_null(State* st);

#ifdef ERL_MAP_SUPPORT
static inline ERL_NIF_TERM parse_object_to_map(State* st);
#endif

static inline void
grow_stack(State *st){
size_t new_offset = 4 * st->offset;
Expand Down Expand Up @@ -163,6 +168,65 @@ parse_object(State* st){
assert(0);
}

#ifdef ERL_MAP_SUPPORT
static inline ERL_NIF_TERM
parse_object_to_map(State* st){
ERL_NIF_TERM *plist, *plist2;
ERL_NIF_TERM p, key, val, pair;
unsigned char c;

st->cur++;
if(look_ah(st) == '}'){
st->cur++;
p = enif_make_new_map(st->env);
plist = &p;
goto ret;
}
size_t stack_off = st->stack_top - st->stack_down;
for(;;){
if(look_ah(st) == '"'){
if((key = parse_string(st))){
if(look_ah(st) == ':'){
st->cur++;
if((val = parse_json(st))){
pair = enif_make_tuple2(st->env, key, val);
push_term(st, pair);
c = look_ah(st);
st->cur++;
if(c == ','){
continue;
}else if(c == '}'){
ERL_NIF_TERM *down = st->stack_down + stack_off;
const ERL_NIF_TERM *tuple;
int arity = 2;
int count = 0;
p = enif_make_new_map(st->env);
plist = &p;
plist2 = malloc(sizeof(ERL_NIF_TERM));
for (count=0; count < (st->stack_top - down); count++) {
enif_get_tuple(st->env, down[count], &arity, &tuple);
enif_make_map_put(st->env, *plist, tuple[0], tuple[1], plist2);
plist = plist2;
plist2 = &p;
}
st->stack_top = down;
goto ret;
}
}
}
}
}
if(!st->error){
st->error = st->priv->am_esyntax;
}
return (ERL_NIF_TERM)0;
}
ret:
return enif_make_tuple2(st->env, st->priv->am_map, *plist);
assert(0);
}
#endif

static inline ERL_NIF_TERM
parse_object_to_record(State* st){
ERL_NIF_TERM record;
Expand Down Expand Up @@ -381,7 +445,18 @@ parse_json(State *st){
ERL_NIF_TERM num;
switch(look_ah(st)){
case '\"' : return parse_string(st);
case '{' : return (st->resource ? parse_object_to_record(st) : parse_object(st));
case '{' :
if (st->resource) {
return parse_object_to_record(st);
}
#ifdef ERL_MAP_SUPPORT
else if (st->format == st->priv->am_map) {
return parse_object_to_map(st);
}
#endif
else {
return parse_object(st);
};
case '[' : return parse_array(st);
case 't' : return parse_true(st);
case 'f' : return parse_false(st);
Expand Down
72 changes: 71 additions & 1 deletion c_src/encoder.c
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ static inline int match_json(ErlNifEnv* env, ERL_NIF_TERM term, State *st);
static inline int match_tuple(ErlNifEnv* env, ERL_NIF_TERM term, State *st);
static inline int match_term(ErlNifEnv* env, ERL_NIF_TERM term, State *st);

#ifdef ERL_MAP_SUPPORT
static inline int match_empty_map(ErlNifEnv* env, ERL_NIF_TERM term, State *st);
static inline int match_map(ErlNifEnv* env, ERL_NIF_TERM term, State *st);
#endif

static void
do_reserve(size_t sz, State *st){
size_t used = st->cur - st->bin.data;
Expand Down Expand Up @@ -178,6 +183,28 @@ match_empty_list(ErlNifEnv* env, ERL_NIF_TERM term, State *st){
return 1;
}

#ifdef ERL_MAP_SUPPORT
static inline int
match_empty_map(ErlNifEnv* env, ERL_NIF_TERM term, State *st){
if(!enif_is_map(env, term)){
return 0;
}

size_t size;

if(!enif_get_map_size(env, term, &size)){
return 0;
}

if (size > 0){
return 0;
}

b_putc2('{', '}', st);
return 1;
}
#endif

static inline int
match_string(ErlNifEnv* env, ERL_NIF_TERM term, State *st){
if(match_binary(env, term, st)){
Expand Down Expand Up @@ -253,6 +280,41 @@ match_list(ErlNifEnv* env, ERL_NIF_TERM term, State *st){
return 1;
}

#ifdef ERL_MAP_SUPPORT
static inline int
match_map(ErlNifEnv* env, ERL_NIF_TERM term, State *st){
ERL_NIF_TERM map, key, value;
ErlNifMapIterator iter;
//ErlNifMapIteratorEntry element;

if(!enif_is_map(env, term))
return 0;

b_putc('{', st);
map = term;
if (!enif_map_iterator_create(env, map, &iter, ERL_NIF_MAP_ITERATOR_HEAD))
return 0;

do {
if (!enif_map_iterator_get_pair(env, &iter, &key, &value)) {
return 0;
}
if(match_string(env, key, st)){
b_putc(':', st);
if(match_term(env, value, st)){
b_putc(',', st);
continue;
}
}
return 0;

} while (enif_map_iterator_next(env, &iter) && !enif_map_iterator_is_tail(env, &iter));
b_unputc(st); // delete tailing ',';
b_putc('}', st);
enif_map_iterator_destroy(env, &iter);
return 1;
}
#endif

static inline int
match_json(ErlNifEnv* env, ERL_NIF_TERM term, State *st){
Expand Down Expand Up @@ -352,7 +414,15 @@ match_tuple(ErlNifEnv* env, ERL_NIF_TERM term, State *st){
return 1;
}else if(match_list(env, term, st)){
return 1;
}else if(match_proplist(env, term, st)){
}
#ifdef ERL_MAP_SUPPORT
else if(match_empty_map(env, term, st)){
return 1;
}else if(match_map(env, term, st)){
return 1;
}
#endif
else if(match_proplist(env, term, st)){
return 1;
}else if(match_tuple(env, term, st)){
return 1;
Expand Down
1 change: 1 addition & 0 deletions c_src/jsonx.c
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ load(ErlNifEnv* env, void** priv_data, ERL_NIF_TERM load_info){
if(!enif_make_existing_atom(env, "struct", &(pdata->am_struct), ERL_NIF_LATIN1)) return 1;
if(!enif_make_existing_atom(env, "proplist", &(pdata->am_proplist), ERL_NIF_LATIN1)) return 1;
if(!enif_make_existing_atom(env, "eep18", &(pdata->am_eep18), ERL_NIF_LATIN1)) return 1;
if(!enif_make_existing_atom(env, "map", &(pdata->am_map), ERL_NIF_LATIN1)) return 1;
if(!enif_make_existing_atom(env, "no_match", &(pdata->am_no_match), ERL_NIF_LATIN1)) return 1;

*priv_data = (void*)pdata;
Expand Down
5 changes: 5 additions & 0 deletions c_src/jsonx.h
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,10 @@
#define inline __inline
#endif

#if ERL_NIF_MAJOR_VERSION >= 2 && ERL_NIF_MINOR_VERSION >= 7
#define ERL_MAP_SUPPORT
#endif

typedef struct{
ERL_NIF_TERM am_true;
ERL_NIF_TERM am_false;
Expand All @@ -22,6 +26,7 @@ typedef struct{
ERL_NIF_TERM am_struct;
ERL_NIF_TERM am_proplist;
ERL_NIF_TERM am_eep18;
ERL_NIF_TERM am_map;
ERL_NIF_TERM am_no_match;

ErlNifResourceType* encoder_RSTYPE;
Expand Down
9 changes: 2 additions & 7 deletions rebar.config
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
%%% -*- mode: erlang -*-

%% Erlang compiler options
{erl_opts, [debug_info,
warnings_as_errors,
warn_export_all
%% warn_untyped_record
]}.

{erl_opts, [
debug_info,
warnings_as_errors,
warn_export_all
warn_export_all,
{platform_define, "R1(1|2|3|4|5|6)", 'JSONX_NO_MAPS'}
]}.

{xref_checks, [
Expand Down
16 changes: 13 additions & 3 deletions src/jsonx.erl
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
%% <li>any other atom -> string</li>
%% <li>binary -> string</li>
%% <li>number -> number</li>
%% <li>map -> object</li>
%% <li>{struct, PropList} -> object</li>
%% <li>{PropList} -> object</li>
%% <li>PropList -> object</li>
Expand Down Expand Up @@ -79,7 +80,7 @@ decode(JSON) ->
%%@doc Decode JSON to Erlang term with options.
-spec decode(JSON, OPTIONS) -> JSON_TERM when
JSON :: binary(),
OPTIONS :: [{format, struct|eep18|proplist}],
OPTIONS :: [{format, struct|eep18|map|proplist}],
JSON_TERM :: any().
decode(JSON, Options) ->
case parse_format(Options) of
Expand All @@ -99,7 +100,7 @@ decoder(Records_desc) ->
%%@doc Build a JSON decoder with output undefined objects.
-spec decoder(RECORDS_DESC, OPTIONS) -> DECODER when
RECORDS_DESC :: [{tag, [names]}],
OPTIONS :: [{format, struct|eep18|proplist}],
OPTIONS :: [{format, struct|eep18|map|proplist}],
DECODER :: function().
decoder(Records_desc, Options) ->
{RecCnt, UKeyCnt, KeyCnt, UKeys, Keys, Records3} = prepare_for_dec(Records_desc),
Expand Down Expand Up @@ -144,9 +145,18 @@ parse_format([{format, proplist} | _]) ->
proplist;
parse_format([{format, eep18} | _]) ->
eep18;
parse_format([{format, map} | _]) ->
maybe_map();
parse_format([_H | T]) ->
parse_format(T).


-ifndef(JSONX_NO_MAPS).
maybe_map() -> map.
-else.
maybe_map() -> undefined.
-endif.

%%%% Internal for decoder

prepare_for_dec(Records) ->
Expand Down Expand Up @@ -229,7 +239,7 @@ init() ->
Dir ->
filename:join(Dir, ?LIBNAME)
end,
ok = erlang:load_nif(So, [[json, struct, proplist, eep18, no_match], [true, false, null],
ok = erlang:load_nif(So, [[json, struct, proplist, eep18, map, no_match], [true, false, null],
[error, big_num, invalid_string, invalid_json, trailing_data, undefined_record]]).

not_loaded(Line) ->
Expand Down
16 changes: 16 additions & 0 deletions test/map_tests.erl
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
-module(map_tests).
-include_lib("eunit/include/eunit.hrl").
-ifndef(JSONX_NO_MAPS).
%% Test encode map
encl0_test() -> <<"{}">> = jsonx:encode(#{}).
encl1_test() -> <<"{}">> = jsonx:encode(#{}).
encl2_test() -> <<"{\"a\":{\"b\":{\"c\":3}}}">> = jsonx:encode(#{a=>#{b=>#{c=>3}}}).
encl3_test() -> <<"{\"a\":2,\"b\":3,\"c\":4,\"d\":5}">> = jsonx:encode(#{a=>2, b=>3, c=>4, d=>5}).

%%% Test decode map
decarr0_test() ->
{map, #{}} = jsonx:decode(<<"{}">>, [{format, map}]).
decarr1_test() ->
{map,#{<<"a">> := 2, <<"b">> := 3, <<"c">> := 4, <<"d">> := 5}} =
jsonx:decode(<<"{\"a\":2,\"b\":3,\"c\":4,\"d\":5}">>, [{format, map}]).
-endif.