Skip to content

Commit

Permalink
* XML<>JSON conversion problems [#66]
Browse files Browse the repository at this point in the history 
  * CDATA sections stripped from XML when converted to JSON
  • Loading branch information
@olofhagsand
olofhagsand committed Jan 19, 2019
1 parent 906b93c commit 19f26e6
Show file tree
Hide file tree
Showing 4 changed files with 148 additions and 50 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,8 @@
* <!DOCTYPE (ie DTD) is not supported.
### Corrected Bugs
* XML<>JSON conversion problems [https://github.com/clicon/clixon/issues/66]
* CDATA sections stripped from XML when converted to JSON
* Restconf returns error when RPC generates "ok" reply [https://github.com/clicon/clixon/issues/69]
* xsd regular expression support for character classes [https://github.com/clicon/clixon/issues/68]
* added support for \c, \d, \w, \W, \s, \S.
Expand Down
76 changes: 46 additions & 30 deletions lib/src/clixon_json.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -216,42 +216,58 @@ array_eval(cxobj *xprev,
return array;
}

/*! Escape a json string
/*! Escape a json string as well as decode xml cdata
* And a
*/
static char *
json_str_escape(char *str)
static int
json_str_escape_cdata(cbuf *cb,
char *str)
{
int i, j;
char *snew;
int retval = -1;
char *snew = NULL;
int i;
int esc = 0; /* cdata escape */

j = 0;
for (i=0;i<strlen(str);i++)
switch (str[i]){
case '\n':
cprintf(cb, "\\n");
break;
case '\"':
cprintf(cb, "\\\"");
break;
case '\\':
j++;
cprintf(cb, "\\\\");
break;
}
if ((snew = malloc(strlen(str)+1+j))==NULL){
clicon_err(OE_XML, errno, "malloc");
return NULL;
}
j = 0;
for (i=0;i<strlen(str);i++)
switch (str[i]){
case '\n':
snew[j++]='\\';
snew[j++]='n';
case '<':
if (!esc &&
strncmp(&str[i], "<![CDATA[", strlen("<![CDATA[")) == 0){
esc=1;
i += strlen("<![CDATA[")-1;
}
else
cprintf(cb, "%c", str[i]);
break;
case ']':
if (esc &&
strncmp(&str[i], "]]>", strlen("]]>")) == 0){
esc=0;
i += strlen("]]>")-1;
}
else
cprintf(cb, "%c", str[i]);
break;
case '\"':
case '\\':
snew[j++]='\\';
default: /* fall thru */
snew[j++]=str[i];
cprintf(cb, "%c", str[i]);
break;
}
snew[j++]='\0';
return snew;
if ((snew = strdup(cbuf_get(cb))) ==NULL){
clicon_err(OE_XML, errno, "strdup");
goto done;
}
retval = 0;
done:
return retval;
}

/*! Do the actual work of translating XML to JSON
Expand Down Expand Up @@ -309,10 +325,10 @@ xml2json1_cbuf(cbuf *cb,
yang_stmt *ymod; /* yang module */
yang_spec *yspec = NULL; /* yang spec */
int bodystr0=1;
char *str;
char *prefix=NULL; /* prefix / local namespace name */
char *namespace=NULL; /* namespace uri */
char *modname=NULL; /* Module name */
int commas;

/* If x is labelled with a default namespace, it should be translated
* to a module name.
Expand All @@ -337,10 +353,11 @@ xml2json1_cbuf(cbuf *cb,
switch(arraytype){
case BODY_ARRAY:{
if (bodystr){
if ((str = json_str_escape(xml_value(x))) == NULL)
/* XXX String if right type */
cprintf(cb, "\"");
if (json_str_escape_cdata(cb, xml_value(x)) < 0)
goto done;
cprintf(cb, "\"%s\"", str);
free(str);
cprintf(cb, "\"");
}
else
cprintf(cb, "%s", xml_value(x));
Expand Down Expand Up @@ -434,8 +451,7 @@ xml2json1_cbuf(cbuf *cb,
break;
}

int na = xml_child_nr_notype(x, CX_ATTR);
int commas = na - 1;
commas = xml_child_nr_notype(x, CX_ATTR) - 1;
for (i=0; i<xml_child_nr(x); i++){
xc = xml_child_i(x, i);
if (xml_type(xc) == CX_ATTR)
Expand Down
59 changes: 57 additions & 2 deletions test/lib.sh
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
#!/bin/bash
# Define test functions.
# Create working dir as variable "dir"
# The functions are somewhat wildgrown, a little too many:
# - expectfn
# - expecteq
# - expecteof
# - expecteofx
# - expecteof_file
# - expectwait
# - expectmatch

#set -e

Expand Down Expand Up @@ -160,11 +168,53 @@ expecteq(){
# - expected command return value (0 if OK)
# - stdin input
# - expected stdout outcome
# Use this if you want regex eg ^foo$
expecteof(){
cmd=$1
retval=$2
input=$3
expect=$4
# Do while read stuff
ret=$($cmd<<EOF
$input
EOF
)
r=$?
if [ $r != $retval ]; then
echo -e "\e[31m\nError ($r != $retval) in Test$testnr [$testname]:"
echo -e "\e[0m:"
exit -1
fi
# If error dont match output strings (why not?)
# if [ $r != 0 ]; then
# return
# fi
# Match if both are empty string
if [ -z "$ret" -a -z "$expect" ]; then
return
fi
# -E for regexp (eg ^$). -Z for nul character, -x for implicit ^$ -q for quiet
# -o only matching
# Two variants: -EZo and -Fxq
# match=`echo "$ret" | grep -FZo "$expect"`
r=$(echo "$ret" | grep -GZo "$expect")
match=$?
# echo "r:\"$r\""
# echo "ret:\"$ret\""
# echo "expect:\"$expect\""
# echo "match:\"$match\""
if [ $match -ne 0 ]; then
err "$expect" "$ret"
fi
}

# Like expecteof but with grep -Fxq instead of -EZq. Ie implicit ^$
# Use this for fixed all line, ie must match exact.
expecteofx(){
cmd=$1
retval=$2
input=$3
expect=$4

# Do while read stuff
ret=$($cmd<<EOF
Expand All @@ -185,11 +235,16 @@ EOF
if [ -z "$ret" -a -z "$expect" ]; then
return
fi
match=`echo "$ret" | grep -GZo "$expect"`
# -E for regexp (eg ^$). -Z for nul character, -x for implicit ^$ -q for quiet
# -o only matching
# Two variants: -EZo and -Fxq
# match=`echo "$ret" | grep -FZo "$expect"`
r=$(echo "$ret" | grep -Fxq "$expect")
match=$?
# echo "ret:\"$ret\""
# echo "expect:\"$expect\""
# echo "match:\"$match\""
if [ -z "$match" ]; then
if [ $match -ne 0 ]; then
err "$expect" "$ret"
fi
}
Expand Down
61 changes: 43 additions & 18 deletions test/test_xml.sh
View file
Original file line number Diff line number Diff line change
@@ -1,24 +1,22 @@
#!/bin/bash
# Test: XML parser tests
# Test: XML parser tests and JSON translation
# @see https://www.w3.org/TR/2008/REC-xml-20081126
# https://www.w3.org/TR/2009/REC-xml-names-20091208
#PROG="valgrind --leak-check=full --show-leak-kinds=all ../util/clixon_util_xml"


# include err() and new() functions and creates $dir
. ./lib.sh


PROG="../util/clixon_util_xml -D $DBG"

new "xml parse"
expecteof "$PROG" 0 "<a><b/></a>" "^<a><b/></a>$"

new "xml parse to json"
expecteof "$PROG -j" 0 "<a><b/></a>" '^{"a": {"b": null}}$'
expecteof "$PROG -j" 0 "<a><b/></a>" '{"a": {"b": null}}'

new "xml parse strange names"
expecteof "$PROG" 0 "<_-><b0.><c-.-._/></b0.></_->" "^<_-><b0.><c-.-._/></b0.></_->$"
expecteof "$PROG" 0 "<_-><b0.><c-.-._/></b0.></_->" "<_-><b0.><c-.-._/></b0.></_->"

new "xml parse name errors"
expecteof "$PROG" 255 "<-a/>" ""
Expand All @@ -37,11 +35,16 @@ if [ "$ret" != "<x>a${LF}b${LF}c${LF}d</x>" ]; then
err '<x>a$LFb$LFc</x>' "$ret"
fi

new "xml simple CDATA"
expecteofx "$PROG" 0 '<a><![CDATA[a text]]></a>' '<a><![CDATA[a text]]></a>'

new "xml simple CDATA to json"
expecteofx "$PROG -j" 0 '<a><![CDATA[a text]]></a>' '{"a": "a text"}'

new "xml complex CDATA"
XML=$(cat <<EOF
<a><description>An example of escaped CENDs</description>
<sometext>
<![CDATA[ They're saying "x < y" & that "z > y" so I guess that means that z > x ]]>
</sometext>
<sometext><![CDATA[ They're saying "x < y" & that "z > y" so I guess that means that z > x ]]></sometext>
<!-- This text contains a CEND ]]> -->
<!-- In this first case we put the ]] at the end of the first CDATA block
and the > in the second CDATA block -->
Expand All @@ -53,33 +56,55 @@ XML=$(cat <<EOF
EOF
)

new "xml CDATA"
expecteof "$PROG" 0 "$XML" "^<a><description>An example of escaped CENDs</description><sometext>
<![CDATA[ They're saying \"x < y\" & that \"z > y\" so I guess that means that z > x ]]>
</sometext><data><![CDATA[This text contains a CEND ]]]]><![CDATA[>]]></data><alternative><![CDATA[This text contains a CEND ]]]><![CDATA[]>]]></alternative></a>$"

JSON=$(cat <<EOF
{"a": {"description": "An example of escaped CENDs","sometext": " They're saying \"x < y\" & that \"z > y\" so I guess that means that z > x ","data": "This text contains a CEND ]]>","alternative": "This text contains a CEND ]]>"}}
EOF
)
new "xml complex CDATA to json"
expecteofx "$PROG -j" 0 "$XML" "$JSON"

XML=$(cat <<EOF
<message>Less than: &lt; , greater than: &gt; ampersand: &amp; </message>
EOF
)
new "xml encode <>&"
expecteof "$PROG" 0 "$XML" "^$XML$"
expecteof "$PROG" 0 "$XML" "$XML"

new "xml encode <>& to json"
expecteof "$PROG -j" 0 "$XML" '{"message": "Less than: < , greater than: > ampersand: & "}'

XML=$(cat <<EOF
<message>To allow attribute values to contain both single and double quotes, the apostrophe or single-quote character ' may be represented as &apos; and the double-quote character as &quot;</message>
<message>single-quote character ' represented as &apos; and double-quote character as &quot;</message>
EOF
)
new "xml single and double quote"
expecteof "$PROG" 0 "$XML" "<message>single-quote character ' represented as ' and double-quote character as \"</message>"

JSON=$(cat <<EOF
{"message": "single-quote character ' represented as ' and double-quote character as \""}
EOF
)
new "xml optional encode single and double quote"
expecteof "$PROG" 0 "$XML" "^<message>To allow attribute values to contain both single and double quotes, the apostrophe or single-quote character ' may be represented as ' and the double-quote character as \"</message>$"
new "xml single and double quotes to json"
expecteofx "$PROG -j" 0 "$XML" "$JSON"

new "xml backspace"
expecteofx "$PROG" 0 "<a>a\b</a>" "<a>a\b</a>"

new "xml backspace to json"
expecteofx "$PROG -j" 0 "<a>a\b</a>" '{"a": "a\\b"}'

new "Double quotes for attributes"
expecteof "$PROG" 0 '<x a="t"/>' '^<x a="t"/>$'
expecteof "$PROG" 0 '<x a="t"/>' '<x a="t"/>'

new "Single quotes for attributes (returns double quotes but at least parses right)"
expecteof "$PROG" 0 "<x a='t'/>" '^<x a="t"/>$'
expecteof "$PROG" 0 "<x a='t'/>" '<x a="t"/>'

new "Mixed quotes"
expecteof "$PROG" 0 "<x a='t' b=\"q\"/>" '^<x a="t" b="q"/>$'
expecteof "$PROG" 0 "<x a='t' b=\"q\"/>" '<x a="t" b="q"/>'

new "XMLdecl version"
expecteof "$PROG" 0 '<?xml version="1.0"?><a/>' '<a/>'
Expand All @@ -94,7 +119,7 @@ new "XMLdecl no version"
expecteof "$PROG" 255 '<?xml ?><a/>' ''

new "XMLdecl misspelled version"
expecteof "$PROG -l o" 255 '<?xml verion="1.0"?><a/>' 'yntax error: at or before: v'
expecteof "$PROG -l o" 255 '<?xml verion="1.0"?><a/>' ''

new "XMLdecl version + encoding"
expecteof "$PROG" 0 '<?xml version="1.0" encoding="UTF-16"?><a/>' '<a/>'
Expand All @@ -119,7 +144,7 @@ expecteof "$PROG" 0 '<?foo something ?><a/><?bar more stuff ?><!-- a comment-->'
#expecteof "$PROG" 255 '<a/><b/>' ''

new "namespace: DefaultAttName"
expecteof "$PROG" 0 '<x xmlns="n1">hello</x>' '^<x xmlns="n1">hello</x>$'
expecteof "$PROG" 0 '<x xmlns="n1">hello</x>' '<x xmlns="n1">hello</x>'

new "namespace: PrefixedAttName"
expecteof "$PROG" 0 '<x xmlns:n2="urn:example:des"><n2:y>hello</n2:y></x>' '^<x xmlns:n2="urn:example:des"><n2:y>hello</n2:y></x>$'
Expand Down

0 comments on commit 19f26e6

Please sign in to comment.