From 19f26e6838f455b074d8cb2a22c4d13574198250 Mon Sep 17 00:00:00 2001
From: Olof hagsand <olof@hagsand.se>
Date: Sat, 19 Jan 2019 21:39:19 +0100
Subject: [PATCH] * XML<>JSON conversion problems
 [https://github.com/clicon/clixon/issues/66]   * CDATA sections stripped from
 XML when converted to JSON

---
 CHANGELOG.md          |  2 ++
 lib/src/clixon_json.c | 76 ++++++++++++++++++++++++++-----------------
 test/lib.sh           | 59 +++++++++++++++++++++++++++++++--
 test/test_xml.sh      | 61 ++++++++++++++++++++++++----------
 4 files changed, 148 insertions(+), 50 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9b2b756a0..088d8a704 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -134,6 +134,8 @@
   * <!DOCTYPE (ie DTD) is not supported.
 
 ### Corrected Bugs
+* XML<>JSON conversion problems [https://github.com/clicon/clixon/issues/66]
+  * CDATA sections stripped from XML when converted to JSON
 * Restconf returns error when RPC generates "ok" reply [https://github.com/clicon/clixon/issues/69]
 * xsd regular expression support for character classes [https://github.com/clicon/clixon/issues/68]
   * added support for \c, \d, \w, \W, \s, \S.
diff --git a/lib/src/clixon_json.c b/lib/src/clixon_json.c
index 731d90de8..4b412e208 100644
--- a/lib/src/clixon_json.c
+++ b/lib/src/clixon_json.c
@@ -216,42 +216,58 @@ array_eval(cxobj *xprev,
     return array;
 }
 
-/*! Escape a json string
+/*! Escape a json string as well as decode xml cdata
+ * And a
  */
-static char *
-json_str_escape(char *str)
+static int
+json_str_escape_cdata(cbuf *cb,
+		      char *str)
 {
-    int   i, j;
-    char *snew;
+    int   retval = -1;
+    char *snew = NULL;
+    int   i;
+    int   esc = 0; /* cdata escape */
 
-    j = 0;
     for (i=0;i<strlen(str);i++)
 	switch (str[i]){
 	case '\n':
+	    cprintf(cb, "\\n");
+	    break;
 	case '\"':
+	    cprintf(cb, "\\\"");
+	    break;
 	case '\\':
-	    j++;
+	    cprintf(cb, "\\\\");
 	    break;
-	}
-    if ((snew = malloc(strlen(str)+1+j))==NULL){
-	clicon_err(OE_XML, errno, "malloc");
-	return NULL;
-    }
-    j = 0;
-    for (i=0;i<strlen(str);i++)
-	switch (str[i]){
-	case '\n':
-	    snew[j++]='\\';
-	    snew[j++]='n';
+	case '<':
+	    if (!esc &&
+		strncmp(&str[i], "<![CDATA[", strlen("<![CDATA[")) == 0){
+		esc=1;
+		i += strlen("<![CDATA[")-1;
+	    }
+	    else
+		cprintf(cb, "%c", str[i]);
+	    break;
+	case ']':
+	    if (esc &&
+		strncmp(&str[i], "]]>", strlen("]]>")) == 0){
+		esc=0;
+		i += strlen("]]>")-1;
+	    }
+	    else
+		cprintf(cb, "%c", str[i]);
 	    break;
-	case '\"':
-	case '\\':
-	    snew[j++]='\\';
 	default: /* fall thru */
-	    snew[j++]=str[i];
+	    cprintf(cb, "%c", str[i]);
+	    break;
 	}
-    snew[j++]='\0';
-    return snew;
+    if ((snew = strdup(cbuf_get(cb))) ==NULL){
+	clicon_err(OE_XML, errno, "strdup");
+	goto done;
+    }
+    retval = 0;
+ done:
+    return retval;
 }
 
 /*! Do the actual work of translating XML to JSON 
@@ -309,10 +325,10 @@ xml2json1_cbuf(cbuf                   *cb,
     yang_stmt       *ymod; /* yang module */
     yang_spec       *yspec = NULL; /* yang spec */
     int              bodystr0=1;
-    char            *str;
     char            *prefix=NULL;    /* prefix / local namespace name */
     char            *namespace=NULL; /* namespace uri */
     char            *modname=NULL;   /* Module name */
+    int              commas;
 
     /* If x is labelled with a default namespace, it should be translated
      * to a module name. 
@@ -337,10 +353,11 @@ xml2json1_cbuf(cbuf                   *cb,
     switch(arraytype){
     case BODY_ARRAY:{
 	if (bodystr){
-	    if ((str = json_str_escape(xml_value(x))) == NULL)
+	    /* XXX String if right type */
+	    cprintf(cb, "\"");
+	    if (json_str_escape_cdata(cb, xml_value(x)) < 0)
 		goto done;
-	    cprintf(cb, "\"%s\"", str);
-	    free(str);
+	    cprintf(cb, "\"");
 	}
 	else
 	    cprintf(cb, "%s", xml_value(x));
@@ -434,8 +451,7 @@ xml2json1_cbuf(cbuf                   *cb,
 	    break;
 	}
 
-    int na = xml_child_nr_notype(x, CX_ATTR);
-    int commas = na - 1;
+    commas = xml_child_nr_notype(x, CX_ATTR) - 1;
     for (i=0; i<xml_child_nr(x); i++){
 	xc = xml_child_i(x, i);
 	if (xml_type(xc) == CX_ATTR)
diff --git a/test/lib.sh b/test/lib.sh
index c18e76fcb..0c0d47dc4 100755
--- a/test/lib.sh
+++ b/test/lib.sh
@@ -1,6 +1,14 @@
 #!/bin/bash
 # Define test functions.
 # Create working dir as variable "dir"
+# The functions are somewhat wildgrown, a little too many:
+# - expectfn
+# - expecteq
+# - expecteof
+# - expecteofx
+# - expecteof_file
+# - expectwait
+# - expectmatch
 
 #set -e
 
@@ -160,11 +168,53 @@ expecteq(){
 # - expected command return value (0 if OK)
 # - stdin input
 # - expected stdout outcome
+# Use this if you want regex eg  ^foo$
 expecteof(){
   cmd=$1
   retval=$2
   input=$3
   expect=$4
+# Do while read stuff
+ret=$($cmd<<EOF 
+$input
+EOF
+)
+  r=$? 
+  if [ $r != $retval ]; then
+      echo -e "\e[31m\nError ($r != $retval) in Test$testnr [$testname]:"
+      echo -e "\e[0m:"
+      exit -1
+  fi
+  # If error dont match output strings (why not?)
+#  if [ $r != 0 ]; then
+#      return
+#  fi
+  # Match if both are empty string
+  if [ -z "$ret" -a -z "$expect" ]; then
+      return
+  fi
+  # -E for regexp (eg ^$). -Z for nul character, -x for implicit ^$ -q for quiet
+  # -o only matching
+  # Two variants: -EZo and -Fxq
+  #  match=`echo "$ret" | grep -FZo "$expect"`
+  r=$(echo "$ret" | grep -GZo "$expect")
+  match=$?
+#  echo "r:\"$r\""
+#  echo "ret:\"$ret\""
+#  echo "expect:\"$expect\""
+#  echo "match:\"$match\""
+  if [ $match -ne 0 ]; then
+      err "$expect" "$ret"
+  fi
+}
+
+# Like expecteof but with grep -Fxq instead of -EZq. Ie implicit ^$
+# Use this for fixed all line, ie must match exact.
+expecteofx(){
+  cmd=$1
+  retval=$2
+  input=$3
+  expect=$4
 
 # Do while read stuff
 ret=$($cmd<<EOF 
@@ -185,11 +235,16 @@ EOF
   if [ -z "$ret" -a -z "$expect" ]; then
       return
   fi
-  match=`echo "$ret" | grep -GZo "$expect"`
+  # -E for regexp (eg ^$). -Z for nul character, -x for implicit ^$ -q for quiet
+  # -o only matching
+  # Two variants: -EZo and -Fxq
+  #  match=`echo "$ret" | grep -FZo "$expect"`
+  r=$(echo "$ret" | grep -Fxq "$expect")
+  match=$?
 #  echo "ret:\"$ret\""
 #  echo "expect:\"$expect\""
 #  echo "match:\"$match\""
-  if [ -z "$match" ]; then
+  if [ $match -ne 0 ]; then
       err "$expect" "$ret"
   fi
 }
diff --git a/test/test_xml.sh b/test/test_xml.sh
index 170c4d92a..10267443c 100755
--- a/test/test_xml.sh
+++ b/test/test_xml.sh
@@ -1,24 +1,22 @@
 #!/bin/bash
-# Test: XML parser tests
+# Test: XML parser tests and JSON translation
 #  @see https://www.w3.org/TR/2008/REC-xml-20081126
 #       https://www.w3.org/TR/2009/REC-xml-names-20091208
 #PROG="valgrind --leak-check=full --show-leak-kinds=all ../util/clixon_util_xml"
 
-
 # include err() and new() functions and creates $dir
 . ./lib.sh
 
-
 PROG="../util/clixon_util_xml -D $DBG"
 
 new "xml parse"
 expecteof "$PROG" 0 "<a><b/></a>" "^<a><b/></a>$"
 
 new "xml parse to json"
-expecteof "$PROG -j" 0 "<a><b/></a>" '^{"a": {"b": null}}$'
+expecteof "$PROG -j" 0 "<a><b/></a>" '{"a": {"b": null}}'
 
 new "xml parse strange names"
-expecteof "$PROG" 0 "<_-><b0.><c-.-._/></b0.></_->" "^<_-><b0.><c-.-._/></b0.></_->$"
+expecteof "$PROG" 0 "<_-><b0.><c-.-._/></b0.></_->" "<_-><b0.><c-.-._/></b0.></_->"
 
 new "xml parse name errors"
 expecteof "$PROG" 255 "<-a/>" ""
@@ -37,11 +35,16 @@ if [ "$ret" != "<x>a${LF}b${LF}c${LF}d</x>" ]; then
      err '<x>a$LFb$LFc</x>' "$ret"
 fi
 
+new "xml simple CDATA"
+expecteofx "$PROG" 0 '<a><![CDATA[a text]]></a>' '<a><![CDATA[a text]]></a>'
+
+new "xml simple CDATA to json"
+expecteofx "$PROG -j" 0 '<a><![CDATA[a text]]></a>' '{"a": "a text"}' 
+
+new "xml complex CDATA"
 XML=$(cat <<EOF
 <a><description>An example of escaped CENDs</description>
-<sometext>
-<![CDATA[ They're saying "x < y" & that "z > y" so I guess that means that z > x ]]>
-</sometext>
+<sometext><![CDATA[ They're saying "x < y" & that "z > y" so I guess that means that z > x ]]></sometext>
 <!-- This text contains a CEND ]]> -->
 <!-- In this first case we put the ]] at the end of the first CDATA block
      and the > in the second CDATA block -->
@@ -53,33 +56,55 @@ XML=$(cat <<EOF
 EOF
 )
 
-new "xml CDATA"
 expecteof "$PROG" 0 "$XML" "^<a><description>An example of escaped CENDs</description><sometext>
 <![CDATA[ They're saying \"x < y\" & that \"z > y\" so I guess that means that z > x ]]>
 </sometext><data><![CDATA[This text contains a CEND ]]]]><![CDATA[>]]></data><alternative><![CDATA[This text contains a CEND ]]]><![CDATA[]>]]></alternative></a>$"
 
+JSON=$(cat <<EOF
+{"a": {"description": "An example of escaped CENDs","sometext": " They're saying \"x < y\" & that \"z > y\" so I guess that means that z > x ","data": "This text contains a CEND ]]>","alternative": "This text contains a CEND ]]>"}}
+EOF
+)       
+new "xml complex CDATA to json"
+expecteofx "$PROG -j" 0 "$XML" "$JSON"
+
 XML=$(cat <<EOF
 <message>Less than: &lt; , greater than: &gt; ampersand: &amp; </message>
 EOF
 )
 new "xml encode <>&"
-expecteof "$PROG" 0 "$XML" "^$XML$"
+expecteof "$PROG" 0 "$XML" "$XML"
+
+new "xml encode <>& to json"
+expecteof "$PROG -j" 0 "$XML" '{"message": "Less than: < , greater than: > ampersand: & "}'
 
 XML=$(cat <<EOF
-<message>To allow attribute values to contain both single and double quotes, the apostrophe or single-quote character ' may be represented as &apos; and the double-quote character as &quot;</message>
+<message>single-quote character ' represented as &apos; and double-quote character as &quot;</message>
+EOF
+)
+new "xml single and double quote"
+expecteof "$PROG" 0 "$XML" "<message>single-quote character ' represented as ' and double-quote character as \"</message>"
+
+JSON=$(cat <<EOF
+{"message": "single-quote character ' represented as ' and double-quote character as \""}
 EOF
 )
-new "xml optional encode single and double quote"
-expecteof "$PROG" 0 "$XML" "^<message>To allow attribute values to contain both single and double quotes, the apostrophe or single-quote character ' may be represented as ' and the double-quote character as \"</message>$"
+new "xml single and double quotes to json"
+expecteofx "$PROG -j" 0 "$XML" "$JSON"
+
+new "xml backspace"
+expecteofx "$PROG" 0 "<a>a\b</a>" "<a>a\b</a>"
+
+new "xml backspace to json"
+expecteofx "$PROG -j" 0 "<a>a\b</a>" '{"a": "a\\b"}'
 
 new "Double quotes for attributes"
-expecteof "$PROG" 0 '<x a="t"/>' '^<x a="t"/>$'
+expecteof "$PROG" 0 '<x a="t"/>' '<x a="t"/>'
 
 new "Single quotes for attributes (returns double quotes but at least parses right)"
-expecteof "$PROG" 0 "<x a='t'/>" '^<x a="t"/>$'
+expecteof "$PROG" 0 "<x a='t'/>" '<x a="t"/>'
 
 new "Mixed quotes"
-expecteof "$PROG" 0 "<x a='t' b=\"q\"/>" '^<x a="t" b="q"/>$'
+expecteof "$PROG" 0 "<x a='t' b=\"q\"/>" '<x a="t" b="q"/>'
 
 new "XMLdecl version"
 expecteof "$PROG" 0 '<?xml version="1.0"?><a/>' '<a/>'
@@ -94,7 +119,7 @@ new "XMLdecl no version"
 expecteof "$PROG" 255 '<?xml ?><a/>' ''
 
 new "XMLdecl misspelled version"
-expecteof "$PROG -l o" 255 '<?xml verion="1.0"?><a/>' 'yntax error: at or before: v'
+expecteof "$PROG -l o" 255 '<?xml verion="1.0"?><a/>' ''
 
 new "XMLdecl version + encoding"
 expecteof "$PROG" 0 '<?xml version="1.0" encoding="UTF-16"?><a/>' '<a/>'
@@ -119,7 +144,7 @@ expecteof "$PROG" 0 '<?foo something ?><a/><?bar more stuff ?><!-- a comment-->'
 #expecteof "$PROG" 255 '<a/><b/>' ''
 
 new "namespace: DefaultAttName"
-expecteof "$PROG" 0 '<x xmlns="n1">hello</x>' '^<x xmlns="n1">hello</x>$'
+expecteof "$PROG" 0 '<x xmlns="n1">hello</x>' '<x xmlns="n1">hello</x>'
 
 new "namespace: PrefixedAttName"
 expecteof "$PROG" 0 '<x xmlns:n2="urn:example:des"><n2:y>hello</n2:y></x>' '^<x xmlns:n2="urn:example:des"><n2:y>hello</n2:y></x>$'