* Optional yangs for testing have been removed from the Clixon repo

  * These were included for testing
  * If you want to run the Clixon test suite you need to point `YANGMODELS`, see test/README.md
  * The following configure options have been removed:
    * `configure --with-opt-yang-installdir=DIR`
    * `configure   --enable-optyangs`
  * You may need to specify standard YANGs using configure option `--with-yang-standard-dir=DIR`
* Updated yang ietf models with fetures for tet
* Added option `CLICON_YANG_AUGMENT_ACCEPT_BROKEN` to accept broken yangmodels.
  * This is a debug option for CI testcases where standard YANG models are broken

configure

@@ -621,8 +621,6 @@ ac_includes_default="\
 
 ac_subst_vars='LTLIBOBJS
 LIBOBJS
-OPT_YANG_INSTALLDIR
-YANG_INSTALLDIR
 EGREP
 GREP
 LEXLIB
@@ -635,8 +633,9 @@ CXXFLAGS
 CXX
 CPP
 wwwdir
+YANG_STANDARD_DIR
+YANG_INSTALLDIR
 CLIXON_YANG_PATCH
-enable_optyangs
 with_libxml2
 HAVE_LIBNGHTTP2
 HAVE_LIBEVHTP
@@ -719,7 +718,6 @@ ac_user_opts='
 enable_option_checking
 enable_debug
 with_cligen
-enable_optyangs
 enable_yang_patch
 enable_publish
 with_restconf
@@ -729,7 +727,7 @@ with_configfile
 with_libxml2
 with_sigaction
 with_yang_installdir
-with_opt_yang_installdir
+with_yang_standard_installdir
 '
       ac_precious_vars='build_alias
 host_alias
@@ -1369,8 +1367,6 @@ Optional Features:
   --disable-FEATURE       do not include FEATURE (same as --enable-FEATURE=no)
   --enable-FEATURE[=ARG]  include FEATURE [ARG=yes]
   --enable-debug          Build with debug symbols, default: no
-  --enable-optyangs       Include optional yang files for examples and testing
-                          in clixon install, default: no
   --enable-yang-patch     Enable YANG patch, RFC 8072, default: no
   --enable-publish        Enable publish of notification streams using SSE and
                           curl
@@ -1393,9 +1389,9 @@ Optional Packages:
   --with-yang-installdir=DIR
                           Install Clixon yang files here (default:
                           ${prefix}/share/clixon)
-  --with-opt-yang-installdir=DIR
-                          Install optional yang files here (default:
-                          ${prefix}/share/clixon)
+  --with-yang-standard-dir=DIR
+                          Where standard IETF/IEEE YANGs are (default:
+                          ${prefix}/share/yang/standard)
 
 Some influential environment variables:
   CC          C compiler command
@@ -3395,7 +3391,9 @@ HAVE_LIBNGHTTP2=false
  # consider using neutral constant such as with-http2
 
 
+# Where Clixon installs its YANG specs
 
+# Examples require standard IETF YANGs. You need to provide these for example and tests
 
 
 # Home dir for web user, such as nginx fcgi sockets
@@ -4590,26 +4588,6 @@ if test -d "${with_cligen}"; then
   test -d "$with_cligen" && CLIGEN_PREFIX="$with_cligen"
 fi
 
-# Disable/enable standard Yang files.
-# If enable - include yang/standard/*.yang in clixon yang files (default)
-# If disable - get standard yang files from elsewhere
-# Check whether --enable-optyangs was given.
-if test "${enable_optyangs+set}" = set; then :
-  enableval=$enable_optyangs;
-	  if test "$enableval" = no; then
-	      enable_optyangs=no
-	  else
-	      enable_optyangs=yes
-          fi
-
-else
-   enable_optyangs=no
-fi
-
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: optyangs is $enable_optyangs" >&5
-$as_echo "optyangs is $enable_optyangs" >&6; }
-
 # Disable/enable yang patch
 # Check whether --enable-yang-patch was given.
 if test "${enable_yang_patch+set}" = set; then :
@@ -5674,26 +5652,23 @@ else
 
 fi
 
-
 { $as_echo "$as_me:${as_lineno-$LINENO}: result: Clixon yang files are installed in ${YANG_INSTALLDIR}" >&5
 $as_echo "Clixon yang files are installed in ${YANG_INSTALLDIR}" >&6; }
 
-# OPT_YANG_INSTALLDIR is where clixon installs standard yang files
-# (  the files in in yang/standard)
-# that Clixon needs to run (or examples rely on). These may be retreived from
-# elsewhere (eg yangmodels repo)
+# YANG_STANDARD_DIR is where clixon assumes standard IETF are
+# This is NOT installed by Clixon and is not needed for core system
+# However, it is required by the main example and some of the tests
 
-# Check whether --with-opt-yang-installdir was given.
-if test "${with_opt_yang_installdir+set}" = set; then :
-  withval=$with_opt_yang_installdir; OPT_YANG_INSTALLDIR="$withval"
+# Check whether --with-yang-standard-installdir was given.
+if test "${with_yang_standard_installdir+set}" = set; then :
+  withval=$with_yang_standard_installdir; YANG_STANDARD_DIR="$withval"
 else
-  OPT_YANG_INSTALLDIR="${prefix}/share/clixon"
+  YANG_STANDARD_DIR="${prefix}/share/yang/standard"
 
 fi
 
-
-{ $as_echo "$as_me:${as_lineno-$LINENO}: result: Optional yang files are installed in ${OPT_YANG_INSTALLDIR} (if enabled)" >&5
-$as_echo "Optional yang files are installed in ${OPT_YANG_INSTALLDIR} (if enabled)" >&6; }
+{ $as_echo "$as_me:${as_lineno-$LINENO}: result: Standard YANG files should be in ${YANG_STANDARD_DIR}" >&5
+$as_echo "Standard YANG files should be in ${YANG_STANDARD_DIR}" >&6; }
 
 # Default location for config file
 
@@ -5704,7 +5679,7 @@ _ACEOF
 
 
 
-ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/config.sh test/cicd/Makefile test/vagrant/Makefile"
+ac_config_files="$ac_config_files Makefile lib/Makefile lib/src/Makefile lib/clixon/Makefile apps/Makefile apps/cli/Makefile apps/backend/Makefile apps/netconf/Makefile apps/restconf/Makefile include/Makefile etc/Makefile etc/clixonrc example/Makefile example/main/Makefile example/main/example.xml extras/rpm/Makefile docker/Makefile docker/main/Makefile docker/base/Makefile util/Makefile yang/Makefile yang/clixon/Makefile yang/mandatory/Makefile yang/optional/Makefile doc/Makefile test/Makefile test/config.sh test/cicd/Makefile test/vagrant/Makefile"
 
 cat >confcache <<\_ACEOF
 # This file is a shell script that caches the results of configure
@@ -6412,6 +6387,7 @@ do
     "etc/clixonrc") CONFIG_FILES="$CONFIG_FILES etc/clixonrc" ;;
     "example/Makefile") CONFIG_FILES="$CONFIG_FILES example/Makefile" ;;
     "example/main/Makefile") CONFIG_FILES="$CONFIG_FILES example/main/Makefile" ;;
+    "example/main/example.xml") CONFIG_FILES="$CONFIG_FILES example/main/example.xml" ;;
     "extras/rpm/Makefile") CONFIG_FILES="$CONFIG_FILES extras/rpm/Makefile" ;;
     "docker/Makefile") CONFIG_FILES="$CONFIG_FILES docker/Makefile" ;;
     "docker/main/Makefile") CONFIG_FILES="$CONFIG_FILES docker/main/Makefile" ;;

configure.ac

@@ -118,9 +118,11 @@ AC_SUBST(with_restconf) # Set to native or fcgi -> compile apps/restconf
 AC_SUBST(HAVE_LIBEVHTP,false)   # consider using neutral constant such as with-http1
 AC_SUBST(HAVE_LIBNGHTTP2,false) # consider using neutral constant such as with-http2
 AC_SUBST(with_libxml2)  
-AC_SUBST(enable_optyangs)
 AC_SUBST(CLIXON_YANG_PATCH)
-
+# Where Clixon installs its YANG specs
+AC_SUBST(YANG_INSTALLDIR) 
+# Examples require standard IETF YANGs. You need to provide these for example and tests
+AC_SUBST(YANG_STANDARD_DIR)
 
 # Home dir for web user, such as nginx fcgi sockets
 AC_SUBST(wwwdir,/www-data)
@@ -173,20 +175,6 @@ if test -d "${with_cligen}"; then
   test -d "$with_cligen" && CLIGEN_PREFIX="$with_cligen"
 fi
 
-# Disable/enable standard Yang files.
-# If enable - include yang/standard/*.yang in clixon yang files (default)
-# If disable - get standard yang files from elsewhere
-AC_ARG_ENABLE(optyangs, AS_HELP_STRING([--enable-optyangs],[Include optional yang files for examples and testing in clixon install, default: no]),[
-	  if test "$enableval" = no; then
-	      enable_optyangs=no
-	  else	      
-	      enable_optyangs=yes
-          fi
-        ],
-	[ enable_optyangs=no])
-
-AC_MSG_RESULT(optyangs is $enable_optyangs)
-
 # Disable/enable yang patch
 AC_ARG_ENABLE(yang-patch, AS_HELP_STRING([--enable-yang-patch],[Enable YANG patch, RFC 8072, default: no]),[
 	  if test "$enableval" = no; then
@@ -347,20 +335,17 @@ AC_ARG_WITH(yang-installdir,
 	    [YANG_INSTALLDIR="$withval"],
     	    [YANG_INSTALLDIR="${prefix}/share/clixon"]		
 	    )
-AC_SUBST(YANG_INSTALLDIR)
 AC_MSG_RESULT(Clixon yang files are installed in ${YANG_INSTALLDIR})	
 
-# OPT_YANG_INSTALLDIR is where clixon installs standard yang files
-# (  the files in in yang/standard)
-# that Clixon needs to run (or examples rely on). These may be retreived from
-# elsewhere (eg yangmodels repo)
-AC_ARG_WITH(opt-yang-installdir,
-	    [AS_HELP_STRING([--with-opt-yang-installdir=DIR],[Install optional yang files here (default: ${prefix}/share/clixon)])],
-	    [OPT_YANG_INSTALLDIR="$withval"],
-    	    [OPT_YANG_INSTALLDIR="${prefix}/share/clixon"]		
+# YANG_STANDARD_DIR is where clixon assumes standard IETF are
+# This is NOT installed by Clixon and is not needed for core system
+# However, it is required by the main example and some of the tests
+AC_ARG_WITH(yang-standard-installdir,
+            [AS_HELP_STRING([--with-yang-standard-dir=DIR],[Where standard IETF/IEEE YANGs are (default: ${prefix}/share/yang/standard)])],
+	    [YANG_STANDARD_DIR="$withval"],
+    	    [YANG_STANDARD_DIR="${prefix}/share/yang/standard"]		
 	    )
-AC_SUBST(OPT_YANG_INSTALLDIR)
-AC_MSG_RESULT(Optional yang files are installed in ${OPT_YANG_INSTALLDIR} (if enabled))	
+AC_MSG_RESULT(Standard YANG files should be in ${YANG_STANDARD_DIR})	
 
 # Default location for config file
 AC_DEFINE_UNQUOTED(CLIXON_DEFAULT_CONFIG,"${CLIXON_DEFAULT_CONFIG}",[Location for apps to find default config file])
@@ -381,6 +366,7 @@ AC_OUTPUT(Makefile
 	  etc/clixonrc
 	  example/Makefile
   	  example/main/Makefile
+    	  example/main/example.xml
 	  extras/rpm/Makefile
 	  docker/Makefile
   	  docker/main/Makefile

doc/FAQ.md

@@ -79,11 +79,10 @@ One of the examples is [a hello world example](https://github.com/clicon/clixon-
 ## How do you build and install Clixon?
 Clixon: 
 ```
-	./configure --enable-optyang;
+	./configure
 	make; 
 	sudo make install; 
 ```
-(note: optyang enable only if you need to run the main example, otherwise it is not necessary).
 
 The main example:
 ```

docker/main/Dockerfile

@@ -39,6 +39,13 @@ MAINTAINER Olof Hagsand <olof@hagsand.se>
 # For clixon and cligen
 RUN apk add --update git make build-base gcc flex bison curl-dev
 
+# Checkut standard YANG models for tests (note >1G for full repo)
+WORKDIR /usr/local/share
+RUN mkdir yang; cd yang; git init; git remote add -f origin https://github.com/YangModels/yang; git config core.sparseCheckout true; echo "standard/" >> .git/info/sparse-checkout; echo "experimental/" >> .git/info/sparse-checkout
+
+WORKDIR /usr/local/share/yang
+RUN git pull origin master
+
 # Create a directory to hold source-code, dependencies etc
 RUN mkdir /clixon
 RUN mkdir /clixon/build
@@ -59,7 +66,7 @@ WORKDIR /clixon/clixon
 COPY clixon .
 
 # Configure, build and install clixon
-RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --enable-optyangs --without-restconf
+RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --without-restconf
 RUN make
 RUN make install
 
@@ -100,6 +107,7 @@ RUN apk add --update sudo curl procps grep make bash # iproute2 # contains ip
 RUN adduser -D -H clicon
 
 COPY --from=0 /clixon/build/ /usr/local/
+COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/standard/
 
 # Log to stderr.
 CMD /usr/local/bin/startsystem.sh

docker/main/Dockerfile.fcgi

@@ -38,6 +38,13 @@ MAINTAINER Olof Hagsand <olof@hagsand.se>
 # For clixon and cligen
 RUN apk add --update git make build-base gcc flex bison fcgi-dev curl-dev
 
+# Checkut standard YANG models for tests (note >1G for full repo)
+WORKDIR /usr/local/share
+RUN mkdir yang; cd yang; git init; git remote add -f origin https://github.com/YangModels/yang; git config core.sparseCheckout true; echo "standard/" >> .git/info/sparse-checkout; echo "experimental/" >> .git/info/sparse-checkout
+
+WORKDIR /usr/local/share/yang
+RUN git pull origin master
+
 # Create a directory to hold source-code, dependencies etc
 RUN mkdir /clixon
 RUN mkdir /clixon/build
@@ -63,7 +70,7 @@ RUN adduser -D -H -G www-data www-data
 RUN apk add --update nginx
 
 # Configure, build and install clixon
-RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --enable-optyangs --with-restconf=fcgi
+RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --with-restconf=fcgi
 RUN make
 RUN make install
 
@@ -114,6 +121,7 @@ RUN adduser nginx clicon
 RUN adduser www-data clicon
 
 COPY --from=0 /clixon/build/ /usr/local/
+COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/standard/
 
 # Manually created
 RUN mkdir /www-data

docker/main/Dockerfile.native

@@ -38,15 +38,30 @@ MAINTAINER Olof Hagsand <olof@hagsand.se>
 # For clixon and cligen
 RUN apk add --update git make build-base gcc flex bison curl-dev
 
-# Create a directory to hold source-code, dependencies etc
-RUN mkdir /clixon
-
 # evhtp dependencies
 RUN apk add --update libevent libevent-dev
 
 # nghttp2 dependencies
 RUN apk add --update nghttp2
 
+# Checkut models
+WORKDIR /usr/local/share/
+
+# Checkut standard YANG models for tests (note >1G for full repo)
+RUN mkdir yang; cd yang; git init; git remote add -f origin https://github.com/YangModels/yang; git config core.sparseCheckout true; echo "standard/" >> .git/info/sparse-checkout; echo "experimental/" >> .git/info/sparse-checkout
+
+WORKDIR /usr/local/share/yang
+RUN git pull origin master
+
+RUN mkdir /usr/local/share/openconfig
+WORKDIR /usr/local/share/openconfig
+
+# Checkut Openconfig models for tests
+RUN git clone https://github.com/openconfig/public
+
+# Create a directory to hold source-code, dependencies etc
+RUN mkdir /clixon
+
 # clone libevhtp
 WORKDIR /clixon
 
@@ -79,7 +94,7 @@ WORKDIR /clixon/clixon
 COPY clixon .
 
 # Configure, build and install clixon
-RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --enable-optyangs --with-restconf=native --enable-nghttp2 --enable-evhtp
+RUN ./configure --prefix=/clixon/build --with-cligen=/clixon/build --with-restconf=native --enable-nghttp2 --enable-evhtp
 
 RUN make
 RUN make install
@@ -134,6 +149,9 @@ RUN adduser -D -H clicon
 
 COPY --from=0 /clixon/build/ /usr/local/
 COPY --from=0 /usr/local/lib/libevhtp.so* /usr/local/lib/
+COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/standard/
+COPY --from=0 /usr/local/share/yang/* /usr/local/share/yang/experimental/
+COPY --from=0 /usr/local/share/openconfig/* /usr/local/share/openconfig/
 
 # Manually created
 RUN mkdir /www-data

docker/main/startsystem.sh

@@ -61,9 +61,13 @@ echo "$STORE" > /usr/local/var/example/running_db
 cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
 SKIPLIST="test_api.sh test_c++.sh test_install.sh test_privileges.sh test_augment.sh test_choice.sh test_identity.sh test_nacm_datanode_read.sh test_nacm_datanode.sh test_nacm_datanode_write.sh test_nacm_default.sh test_nacm_ext.sh test_nacm_module_read.sh test_nacm_module_write.sh test_nacm_protocol.sh test_nacm.sh test_nacm_recovery.sh test_perf.sh test_perf_state_only.sh test_perf_state.sh test_restconf2.sh test_restconf_err.sh test_restconf_jukebox.sh test_restconf_listkey.sh test_restconf_patch.sh test_restconf.sh test_restconf_startup.sh test_rpc.sh test_ssl_certs.sh test_stream.sh test_submodule.sh test_upgrade_auto.sh test_upgrade_interfaces.sh test_upgrade_repair.sh test_yang_namespace.sh"
-#IETFRFC=
+# Parse yangmodels from https://github.com/YangModels/yang
+YANGMODELS=/usr/local/share/yang
 EOF
 
+# Patch yang syntax errors
+sed -i s/=\ olt\'/=\ \'olt\'/g /usr/local/share/yang/standard/ieee/published/802.3/ieee802-ethernet-pon.yang
+
 # Workaround for this error output:
 # sudo: setrlimit(RLIMIT_CORE): Operation not permitted
 echo "Set disable_coredump false" > /etc/sudo.conf

docker/main/startsystem_fcgi.sh

@@ -90,9 +90,13 @@ cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
 SKIPLIST="test_api.sh test_client.sh test_c++.sh test_install.sh test_privileges.sh"
 RCPROTO=http # Because nginx
-#IETFRFC=
+# Parse yangmodels from https://github.com/YangModels/yang
+YANGMODELS=/usr/local/share/yang
 EOF
 
+# Patch yang syntax errors
+sed -i s/=\ olt\'/=\ \'olt\'/g /usr/local/share/yang/standard/ieee/published/802.3/ieee802-ethernet-pon.yang
+
 # Workaround for this error output:
 # sudo: setrlimit(RLIMIT_CORE): Operation not permitted
 echo "Set disable_coredump false" > /etc/sudo.conf

docker/main/startsystem_native.sh

@@ -59,9 +59,13 @@ echo "$STORE" > /usr/local/var/example/running_db
 cat <<EOF > /usr/local/bin/test/site.sh
 # Add your local site specific env variables (or tests) here.
 SKIPLIST="test_api.sh test_client.sh test_c++.sh test_install.sh test_privileges.sh"
-#IETFRFC=
+# Parse yangmodels from https://github.com/YangModels/yang
+YANGMODELS=/usr/local/share/yang
 EOF
 
+# Patch yang syntax errors
+sed -i s/=\ olt\'/=\ \'olt\'/g /usr/local/share/yang/standard/ieee/published/802.3/ieee802-ethernet-pon.yang
+
 # Workaround for this error output:
 # sudo: setrlimit(RLIMIT_CORE): Operation not permitted
 echo "Set disable_coredump false" > /etc/sudo.conf