* Changed first parameter from int fd to FILE *f in the following…

… functions:

  * clixon_xml_parse_file(), clixon_json_parse_file(), yang_parse_file()
  * See [Bytewise read() of files is slow #146](#146)

CHANGELOG.md

@@ -44,8 +44,17 @@ Users may have to change how they access the system
 * New clixon-config@2020-11-03.yang revision
   * Added option: `CLICON_RESTCONF_CONFIG` for reading restconf daemon config frm datastore
 
+### C/CLI-API changes on existing features
+
+Developers may need to change their code
+
+* Changed first parameter from `int fd` to `FILE *f` in the following functions:
+  * clixon_xml_parse_file(), clixon_json_parse_file(), yang_parse_file()
+  * See [Bytewise read() of files is slow #146](https://github.com/clicon/clixon/issues/146)
+
 ### Minor changes
 
+* Improved performance of parsing files as described in [Bytewise read() of files is slow #146](https://github.com/clicon/clixon/issues/146), thanks: @hjelmeland
 * Added new backend plugin: ca_pre-demon called if backend is daemonized just prior to forking.
 * Added XPATH functions `position`
 

apps/backend/backend_main.c

@@ -186,7 +186,6 @@ nacm_load_external(clicon_handle h)
     cxobj      *xt = NULL;
     struct stat st;
     FILE       *f = NULL;
-    int         fd;
 
     filename = clicon_option_str(h, "CLICON_NACM_FILE");
     if (filename == NULL || strlen(filename)==0){
@@ -209,9 +208,8 @@ nacm_load_external(clicon_handle h)
 	goto done;
     if (yang_spec_parse_module(h, "ietf-netconf-acm", NULL, yspec) < 0)
 	goto done;
-    fd = fileno(f);
     /* Read configfile */
-    if (clixon_xml_parse_file(fd, YB_MODULE, yspec, NULL, &xt, NULL) < 0)
+    if (clixon_xml_parse_file(f, YB_MODULE, yspec, NULL, &xt, NULL) < 0)
 	goto done;
     if (xt == NULL){
 	clicon_err(OE_XML, 0, "No xml tree in %s", filename);

apps/backend/backend_startup.c

@@ -168,26 +168,26 @@ load_extraxml(clicon_handle h,
 {
     int        retval =  -1;
     cxobj     *xt = NULL;
-    int        fd = -1;
+    FILE      *fp = NULL;
     yang_stmt *yspec = NULL;
 
     if (filename == NULL)
 	return 1;
-    if ((fd = open(filename, O_RDONLY)) < 0){
+    if ((fp = fopen(filename, "r")) < 0){
 	clicon_err(OE_UNIX, errno, "open(%s)", filename);
 	goto done;
     }
     yspec = clicon_dbspec_yang(h);
-    if (clixon_xml_parse_file(fd, YB_MODULE, yspec, NULL, &xt, NULL) < 0)
+    if (clixon_xml_parse_file(fp, YB_MODULE, yspec, NULL, &xt, NULL) < 0)
 	goto done;
     /* Replace parent w first child */
     if (xml_rootchild(xt, 0, &xt) < 0)
 	goto done;
     /* Merge user reset state */
     retval = xmldb_put(h, (char*)db, OP_MERGE, xt, clicon_username_get(h), cbret);
  done:
-    if (fd != -1)
-	close(fd);
+    if (fp)
+	fclose(fp);
     if (xt)
 	xml_free(xt);
     return retval;

apps/cli/cli_common.c

@@ -771,7 +771,7 @@ load_config_file(clicon_handle h,
     cg_var     *cv;
     char       *opstr;
     char       *varstr;
-    int         fd = -1;
+    FILE       *fp = NULL;
     cxobj      *xt = NULL;
     cxobj      *x;
     cbuf       *cbxml;
@@ -804,11 +804,11 @@ load_config_file(clicon_handle h,
 	goto done;
     }
     /* Open and parse local file into xml */
-    if ((fd = open(filename, O_RDONLY)) < 0){
+    if ((fp = fopen(filename, "r")) < 0){
 	clicon_err(OE_UNIX, errno, "open(%s)", filename);
 	goto done;
     }
-    if (clixon_xml_parse_file(fd, YB_NONE, NULL, NULL, &xt, NULL) < 0)
+    if (clixon_xml_parse_file(fp, YB_NONE, NULL, NULL, &xt, NULL) < 0)
 	goto done;
     if (xt == NULL)
 	goto done;
@@ -831,8 +831,8 @@ load_config_file(clicon_handle h,
  done:
     if (xt)
 	xml_free(xt);
-    if (fd != -1)
-	close(fd);
+    if (fp)
+	fclose(fp);
     return ret;
 }
 

example/main/example_backend.c

@@ -370,7 +370,7 @@ example_statedata(clicon_handle h,
     cvec   *nsc1 = NULL;
     cvec   *nsc2 = NULL;
     yang_stmt *yspec = NULL;
-    int     fd;
+    FILE      *fp = NULL;
 
     if (!_state)
 	goto ok;
@@ -395,15 +395,14 @@ example_statedata(clicon_handle h,
 	}
 	else{
 	    cxobj *x1;
-	    if ((fd = open(_state_file, O_RDONLY)) < 0){
+	    if ((fp = fopen(_state_file, "r")) < 0){
 		clicon_err(OE_UNIX, errno, "open(%s)", _state_file);
 		goto done;
 	    }
 	    if ((xt = xml_new("config", NULL, CX_ELMNT)) == NULL)
 		goto done;
-	    if (clixon_xml_parse_file(fd, YB_MODULE, yspec, NULL, &xt, NULL) < 0)
+	    if (clixon_xml_parse_file(fp, YB_MODULE, yspec, NULL, &xt, NULL) < 0)
 		goto done;
-	    close(fd);
 	    if (xpath_vec(xt, nsc, "%s", &xvec, &xlen, xpath) < 0) 
 		goto done;
 	    for (i=0; i<xlen; i++){
@@ -477,6 +476,8 @@ example_statedata(clicon_handle h,
  ok:
     retval = 0;
  done:
+    if (fp)
+	fclose(fp);
     if (nsc1)
 	xml_nsctx_free(nsc1);
     if (nsc2)
@@ -968,21 +969,20 @@ example_start(clicon_handle h)
 int
 example_daemon(clicon_handle h)
 {
-    int retval = -1;
-    int ret;
+    int        retval = -1;
+    int        ret;
+    FILE      *fp = NULL;
+    yang_stmt *yspec;
 
     /* Read state file (or should this be in init/start?) */
     if (_state && _state_file && _state_file_init){
-	int fd;
-	yang_stmt *yspec = clicon_dbspec_yang(h);
-
-	if ((fd = open(_state_file, O_RDONLY)) < 0){
+	yspec = clicon_dbspec_yang(h);
+	if ((fp = fopen(_state_file, "r")) < 0){
 	    clicon_err(OE_UNIX, errno, "open(%s)", _state_file);
 	    goto done;
 	}
-	if ((ret = clixon_xml_parse_file(fd, YB_MODULE, yspec, NULL, &_state_xstate, NULL)) < 0)
+	if ((ret = clixon_xml_parse_file(fp, YB_MODULE, yspec, NULL, &_state_xstate, NULL)) < 0)
 	    goto done;
-	close(fd);
 	if (ret == 0){
 	    fprintf(stderr, "%s error\n", __FUNCTION__);
 	    goto done;
@@ -991,6 +991,8 @@ example_daemon(clicon_handle h)
     }
     retval = 0;
  done:
+    if (fp)
+	fclose(fp);
     return retval;
 }
 

lib/clixon/clixon_json.h

@@ -51,6 +51,6 @@ int xml2json_cb(FILE *f, cxobj *x, int pretty, clicon_output_cb *fn);
 int json_print(FILE *f, cxobj *x);
 int xml2json_vec(FILE *f, cxobj **vec, size_t veclen, int pretty);
 int clixon_json_parse_string(char *str, yang_bind yb, yang_stmt *yspec, cxobj **xt, cxobj **xret);
-int clixon_json_parse_file(int fd, yang_bind yb, yang_stmt *yspec, cxobj **xt, cxobj **xret);
+int clixon_json_parse_file(FILE *fp, yang_bind yb, yang_stmt *yspec, cxobj **xt, cxobj **xret);
 
 #endif /* _CLIXON_JSON_H */

lib/clixon/clixon_xml_io.h

@@ -50,7 +50,7 @@ int clicon_xml2cbuf(cbuf *cb, cxobj *x, int level, int prettyprint, int32_t dept
 char *clicon_xml2str(cxobj *x);
 int xmltree2cbuf(cbuf *cb, cxobj *x, int level);
 
-int clixon_xml_parse_file(int fd, yang_bind yb, yang_stmt *yspec, char *endtag, cxobj **xt, cxobj **xerr);
+int clixon_xml_parse_file(FILE *f, yang_bind yb, yang_stmt *yspec, char *endtag, cxobj **xt, cxobj **xerr);
 int clixon_xml_parse_string(const char *str, yang_bind yb, yang_stmt *yspec, cxobj **xt, cxobj **xerr);
 
 #if defined(__GNUC__) && __GNUC__ >= 3

lib/clixon/clixon_yang_parse_lib.h

@@ -51,7 +51,7 @@
 /*
  * Prototypes
  */
-yang_stmt *yang_parse_file(int fd, const char *name, yang_stmt *ysp);
+yang_stmt *yang_parse_file(FILE *fp, const char *name, yang_stmt *ysp);
 yang_stmt *yang_parse_filename(const char *filename, yang_stmt  *ysp);
 int        yang_spec_parse_module(clicon_handle h, const char *module,
 				  const char *revision, yang_stmt *yspec);

lib/src/clixon_datastore_read.c

@@ -438,7 +438,7 @@ xmldb_readfile(clicon_handle    h,
     int        retval = -1;
     cxobj     *x0 = NULL;
     char      *dbfile = NULL;
-    int        fd = -1;
+    FILE      *fp = NULL;
     char      *format;
     int        ret;
 
@@ -453,15 +453,15 @@ xmldb_readfile(clicon_handle    h,
 	goto done;
     }
     /* Parse file into internal XML tree from different formats */
-    if ((fd = open(dbfile, O_RDONLY)) < 0) {
+    if ((fp = fopen(dbfile, "r")) < 0) {
 	clicon_err(OE_UNIX, errno, "open(%s)", dbfile);
 	goto done;
     }    
     if (strcmp(format, "json")==0){
-	if ((ret = clixon_json_parse_file(fd, yb, yspec, &x0, NULL)) < 0) /* XXX: ret == 0*/
+	if ((ret = clixon_json_parse_file(fp, yb, yspec, &x0, NULL)) < 0) /* XXX: ret == 0*/
 	    goto done;
     }
-    else if ((ret = clixon_xml_parse_file(fd, yb, yspec, "</config>", &x0, NULL)) < 0)
+    else if ((ret = clixon_xml_parse_file(fp, yb, yspec, "</config>", &x0, NULL)) < 0)
 	goto done;
 #ifdef XMLDB_READFILE_FAIL /* The functions calling this function cannot handle a failed parse yet */
     if (ret == 0)
@@ -495,8 +495,8 @@ xmldb_readfile(clicon_handle    h,
     }
     retval = 1;
  done:
-    if (fd != -1)
-	close(fd);
+    if (fp)
+	fclose(fp);
     if (dbfile)
 	free(dbfile);
     if (x0)

lib/src/clixon_json.c

@@ -1367,14 +1367,14 @@ clixon_json_parse_string(char      *str,
  *   eg: name="a:b" -> prefix="a", name="b"
  * But this is not done if yspec=NULL, and is not part of the JSON spec
  * 
- * @param[in]     fd    File descriptor to the JSON file (ASCII string)
+ * @param[in]     fp    File descriptor to the JSON file (ASCII string)
  * @param[in]     yspec Yang specification, or NULL
  * @param[in,out] xt    Pointer to (XML) parse tree. If empty, create.
  * @param[out]    xerr  Reason for invalid returned as netconf err msg 
  *
  * @code
  *  cxobj *xt = NULL;
- *  if (clixon_json_parse_file(0, YB_MODULE, yspec, &xt) < 0)
+ *  if (clixon_json_parse_file(stdin, YB_MODULE, yspec, &xt) < 0)
  *    err;
  *  xml_free(xt);
  * @endcode
@@ -1390,7 +1390,7 @@ clixon_json_parse_string(char      *str,
  * @see RFC7951
  */
 int
-clixon_json_parse_file(int        fd,
+clixon_json_parse_file(FILE      *fp,
 		       yang_bind  yb,
 		       yang_stmt *yspec,
 		       cxobj    **xt,
@@ -1404,7 +1404,7 @@ clixon_json_parse_file(int        fd,
     char     *ptr;
     char      ch;
     int       len = 0;
-    
+
     if (xt==NULL){
 	clicon_err(OE_XML, EINVAL, "xt is NULL");
 	return -1;
@@ -1416,7 +1416,7 @@ clixon_json_parse_file(int        fd,
     memset(jsonbuf, 0, jsonbuflen);
     ptr = jsonbuf;
     while (1){
-	if ((ret = read(fd, &ch, 1)) < 0){
+	if ((ret = fread(&ch, 1, 1, fp)) < 0){
 	    clicon_err(OE_XML, errno, "read");
 	    break;
 	}

lib/src/clixon_options.c

@@ -204,19 +204,19 @@ parse_configfile_one(const char *filename,
 		     cxobj     **xconfig)
 {
     int    retval = -1;
-    int    fd = -1;
+    FILE  *fp = NULL;
     cxobj *xt = NULL;
     cxobj *xerr = NULL;
     cxobj *xa;
     cbuf  *cbret = NULL;
     int    ret;
 
-    if ((fd = open(filename, O_RDONLY)) < 0){
+    if ((fp = fopen(filename, "r")) < 0){
 	clicon_err(OE_UNIX, errno, "open configure file: %s", filename);
 	return -1;
     }
     clicon_debug(2, "%s: Reading config file %s", __FUNCTION__, filename);
-    if ((ret = clixon_xml_parse_file(fd, yspec?YB_MODULE:YB_NONE, yspec, NULL, &xt, &xerr)) < 0)
+    if ((ret = clixon_xml_parse_file(fp, yspec?YB_MODULE:YB_NONE, yspec, NULL, &xt, &xerr)) < 0)
 	goto done;
     if (ret == 0){
 	if ((cbret = cbuf_new()) ==NULL){
@@ -249,8 +249,8 @@ parse_configfile_one(const char *filename,
  done:
     if (xt)
 	xml_free(xt);
-    if (fd != -1)
-	close(fd);
+    if (fp)
+	fclose(fp);
     if (cbret)
 	cbuf_free(cbret);
     if (xerr)

lib/src/clixon_xml_changelog.c

@@ -436,7 +436,7 @@ clixon_xml_changelog_init(clicon_handle h)
 {
     int        retval = -1;
     char      *filename;
-    int        fd = -1;
+    FILE      *fp = NULL;
     cxobj     *xt = NULL;
     yang_stmt *yspec;
     int        ret;
@@ -445,11 +445,11 @@ clixon_xml_changelog_init(clicon_handle h)
 
     yspec = clicon_dbspec_yang(h);
     if ((filename = clicon_option_str(h, "CLICON_XML_CHANGELOG_FILE")) != NULL){
-	if ((fd = open(filename, O_RDONLY)) < 0){
-	    clicon_err(OE_UNIX, errno, "open(%s)", filename);
+	if ((fp = fopen(filename, "r")) < 0){
+	    clicon_err(OE_UNIX, errno, "fopen(%s)", filename);
 	    goto done;
 	}    
-	if (clixon_xml_parse_file(fd, YB_MODULE, yspec, NULL, &xt, NULL) < 0)
+	if (clixon_xml_parse_file(fp, YB_MODULE, yspec, NULL, &xt, NULL) < 0)
 	    goto done;
 	if (xml_rootchild(xt, 0, &xt) < 0)
 	    goto done;
@@ -477,8 +477,8 @@ clixon_xml_changelog_init(clicon_handle h)
 	cbuf_free(cbret);
     if (xret)
 	xml_free(xret);
-    if (fd != -1)
-	close(fd);
+    if (fp)
+	fclose(fp);
     if (xt)
 	xml_free(xt);
     return retval;

lib/src/clixon_xml_io.c

@@ -578,9 +578,9 @@ FSM(char *tag,
  * @code
  *  cxobj *xt = NULL;
  *  cxobj *xerr = NULL;
- *  int    fd;
- *  fd = open(filename, O_RDONLY);
- *  if ((ret = clixon_xml_parse_file(fd, YB_MODULE, yspec, "</config>", &xt, &xerr)) < 0)
+ *  FILE  *f;
+ *  f = fopen(filename, "r");
+ *  if ((ret = clixon_xml_parse_file(f, YB_MODULE, yspec, "</config>", &xt, &xerr)) < 0)
  *    err;
  *  xml_free(xt);
  * @endcode
@@ -590,7 +590,7 @@ FSM(char *tag,
  * @note May block on file I/O
  */
 int 
-clixon_xml_parse_file(int        fd, 
+clixon_xml_parse_file(FILE      *fp, 
 		      yang_bind  yb,
 		      yang_stmt *yspec,
 		      char      *endtag,
@@ -626,9 +626,8 @@ clixon_xml_parse_file(int        fd,
     memset(xmlbuf, 0, xmlbuflen);
     ptr = xmlbuf;
     while (1){
-	if ((ret = read(fd, &ch, 1)) < 0){
-	    clicon_err(OE_XML, errno, "read: [pid:%d]", 
-		    (int)getpid());
+	if ((ret = fread(&ch, 1, 1, fp)) < 0){
+	    clicon_err(OE_XML, errno, "read");
 	    break;
 	}
 	if (ret != 0){