Merging release/3.0.0

.gitignore

@@ -22,3 +22,4 @@ ylwrap
 *.o
 *.m4
 .dirstamp
+*.swp

CONTRIBUTING.md

@@ -25,5 +25,5 @@ For build instructions, please refer to [QUICK_START_GUIDE](https://github.com/B
 
 ## Contributor License Agreement
 
-Any company or individual interested in participating in this open source project must sign a Contributor License Agreement (more details [here](https://github.com/BroadbandForum/obuspa/wiki/Contributor-License-Agreement-Details)).
+Any company or individual interested in participating in this open source project must sign a Contributor License Agreement (more details [here](https://wiki.broadband-forum.org/pages/viewpage.action?pageId=87557395)).
 

Makefile.am

@@ -14,8 +14,9 @@ AM_CFLAGS = -Wall -Wfatal-errors
 
 bin_PROGRAMS = obuspa
 
+OBUSPA_LOCAL_STATE_DIR = "$(localstatedir)/obuspa"
 
-obuspa_SOURCES = src/core/main.c \
+SOURCES = src/core/main.c \
                     src/core/mtp_exec.c \
                     src/core/dm_exec.c \
                     src/core/bdc_exec.c \
@@ -31,6 +32,8 @@ obuspa_SOURCES = src/core/main.c \
                     src/core/handle_get_supported_protocol.c \
                     src/core/handle_get_instances.c \
                     src/core/handle_get_supported_dm.c \
+                    src/core/group_get_vector.c \
+                    src/core/group_set_vector.c \
                     src/core/proto_trace.c \
                     src/core/data_model.c \
                     src/core/error_resp.c \
@@ -43,7 +46,6 @@ obuspa_SOURCES = src/core/main.c \
                     src/core/device_stomp.c \
                     src/core/device_subscription.c \
                     src/core/device_security.c \
-                    src/core/device_firmware.c \
                     src/core/device_ctrust.c \
                     src/core/device_bulkdata.c \
                     src/core/device_selftest_example.c \
@@ -78,13 +80,18 @@ obuspa_SOURCES = src/core/main.c \
                     src/protobuf-c/usp-msg.pb-c.c \
                     src/protobuf-c/usp-record.pb-c.c \
                     src/protobuf-c/protobuf-c.c \
-					src/core/coap_common.c \
-					src/core/coap_client.c \
-					src/core/coap_server.c \
-                    src/core/uri.c
+                    src/core/coap_common.c \
+                    src/core/coap_client.c \
+                    src/core/coap_server.c \
+                    src/core/uri.c \
+                    src/core/mqtt.c \
+                    src/core/device_mqtt.c
 
-obuspa_CPPFLAGS = $(openssl_CFLAGS) $(sqlite3_CFLAGS) $(libcurl_CFLAGS) $(libcares_CFLAGS) $(zlib_CFLAGS)
+obuspa_SOURCES = $(SOURCES)
+obuspa_CPPFLAGS = $(openssl_CFLAGS) $(sqlite3_CFLAGS) $(libcurl_CFLAGS) $(zlib_CFLAGS) $(libmosquitto_CFLAGS)
 obuspa_CPPFLAGS += -DENABLE_COAP
+obuspa_CPPFLAGS += -DENABLE_MQTT
+obuspa_CPPFLAGS += -DOBUSPA_LOCAL_STATE_DIR=\"$(OBUSPA_LOCAL_STATE_DIR)\"
 obuspa_CPPFLAGS +=  $(AM_CPPFLAGS) \
                       -Werror \
                       -Werror=unused-value \
@@ -98,9 +105,22 @@ obuspa_CPPFLAGS +=  $(AM_CPPFLAGS) \
 obuspa_LDFLAGS = -rdynamic
 
 obuspa_LDADD = -lm -ldl -lpthread -lrt
-obuspa_LDADD += $(openssl_LIBS) $(sqlite3_LIBS) $(libcurl_LIBS) $(libcares_LIBS) $(zlib_LIBS)
+obuspa_LDADD += $(openssl_LIBS) $(sqlite3_LIBS) $(libcurl_LIBS) $(zlib_LIBS) $(libmosquitto_LIBS)
 
 obuspa_LDFLAGS += -Wl,-rpath=/usr/local/lib
 
+# Create obuspa directory for usp.db etc on install
+# This depends on your prefix setting (default localstatedir=/usr/local/var/)
+# Default OBUSPA_LOCAL_STATE_DIR=/usr/local/var/obuspa
+install-data-local:
+	${mkdir_p} "${DESTDIR}/${OBUSPA_LOCAL_STATE_DIR}"
+	if [ -n "${SUDO_USER}" ]; then \
+		chown "${SUDO_USER}" "${DESTDIR}/${OBUSPA_LOCAL_STATE_DIR}"; \
+	fi
+
+uninstall-local:
+	rm -rf "${DESTDIR}/${OBUSPA_LOCAL_STATE_DIR}"
+
 # Import vendor makefile
 include src/vendor/vendor.am
+

QUICK_START_GUIDE.md

@@ -16,9 +16,11 @@ For example,  DEVICE_XXX_Init() refers to a set of functions:
 
 
 ## Building OB-USP-AGENT
-1. Install dependencies (Curl, OpenSSL, Sqlite, C-Ares, z-lib, autotools) using package manager:
+1. Install dependencies (Curl, OpenSSL, Sqlite, z-lib, autotools) using package manager:
 ```
-$ sudo apt-get install libssl-dev libcurl4-openssl-dev libsqlite3-dev libc-ares-dev libz-dev autoconf automake libtool
+$ sudo apt-add-repository ppa:mosquitto-dev/mosquitto-ppa
+$ sudo apt-get update
+$ sudo apt-get install libssl-dev libcurl4-openssl-dev libsqlite3-dev libz-dev autoconf automake libtool libmosquitto-dev pkg-config
 ```
 
 2. Install OB-USP-AGENT from source:
@@ -64,16 +66,16 @@ If OB-USP-AGENT successfully connected to your STOMP server you should see trace
    endpoint-id:os\c\c002456-0800270B57FF
    login:my_username
    passcode:
-   
-   
+
+
    Received CONNECTED frame from (host=controller1, port=61613)
    CONNECTED
    session:session-1K6NehQnoR3hioRXZgFBBw
    heart-beat:300000,30000
    server:RabbitMQ/3.5.7
    version:1.2
-   
-   
+
+
    Sending SUBSCRIBE frame to (host=controller1, port=61613)
    SUBSCRIBE
    id:0
@@ -86,7 +88,7 @@ If you subsequently change the settings in factory_reset_example.txt, then you m
 in order that the database is re-created the next time you run OB-USP_AGENT.
 To delete the database in the default location:
 ```
-$ rm /tmp/usp.db
+$ rm /usr/local/var/obuspa/usp.db
 ```
 
 Alternatively you can use the 'obuspa -c dbset' command (see next section) to alter parameters
@@ -189,7 +191,7 @@ The following defines are most likely to need modifying:
                                     used for USP communications. If not defined, the Linux routing tables select
                                     which network interface to use.
                                     IMPORTANT: Even if not defined, DEFAULT_WAN_IFNAME must be a valid network interface.
-                                    
+
 * DEFAULT_DATABASE_FILE - The file system location of the database file, if none is specified
                           by the '-f' option when invoking OB-USP-AGENT.
 
@@ -206,7 +208,7 @@ The following defines are most likely to need modifying:
 ## Extending the Data Model
 Use the USP_REGISTER_XXX() set of functions to register USP data model objects, parameters, cammands and Events.
 * Integrators should always call USP_REGISTER_XXX() from VENDOR_Init() in src/vendor/vendor.c
-* Contributors should create a new device_XXX.c file in src/core, and call USP_REGISTER_XX() from a 
+* Contributors should create a new device_XXX.c file in src/core, and call USP_REGISTER_XX() from a
   DEVICE_XXX_Init() located in the new device_XXX.c file.
   The new DEVICE_XXX_Init() must be hooked into the existing core data model from DATA_MODEL_Init() (in src/core/data_model.c).
 
@@ -238,7 +240,7 @@ At bootup, the instance numbers of data model objects must be signalled to OB-US
 * Integrators should call USP_DM_InformInstance() from VENDOR_Start() (in src/vendor/vendor.c).
 * Contributors should call USP_DM_InformInstance() from a DEVICE_XXX_Start() function in their device_XXX.c file.
 
-After bootup, changes to object instances should be signalled with the USP_SIGNAL_ObjectAdded() and 
+After bootup, changes to object instances should be signalled with the USP_SIGNAL_ObjectAdded() and
 USP_SIGNAL_ObjectDeleted() functions.
 
 For an example of implementing a USP asynchronous command, see src/core/device_selftest_example.c.
@@ -284,4 +286,111 @@ The following core vendor hooks are most likely to need overriding:
 * get_trust_store_cb - called by OB-USP-AGENT core to get the list of SSL certificates to install in OB-USP-AGENT's trust store. These can alternatively be specified using the '-t' option when invoking OB-USP-AGENT.
 * get_agent_cert_cb - called by OB-USP-AGENT core to get the SSL client certificate and private key associated with this device. This can alternatively be specified using the '-a' option when invoking OB-USP-AGENT.
 
-Certificates provided to the get_trust_store_cb() and get_agent_cert_cb() must be in DER (binary) form.
+The trust store certificates and agent certificate (with associated private key) may alternatively be specified using the '--truststore' and
+'--authcert' arguments when invoking OB-USP-AGENT.
+
+Certificates provided to the get_trust_store_cb() and get_agent_cert_cb() must be in DER (binary) form,
+whilst certificates provided to the '--authcert' and '--truststore' invocation arguments must be in PEM format.
+
+## Advanced APIs
+### Extending the Data Model using grouped parameter and object API
+With some integrations, the data model is implemented by other executables and OB-USP-AGENT must communicate with the other executables to get or set parameters or add or delete object instances. 
+
+Use the USP_REGISTER_GroupXXX() set of functions to register the group of parameters and objects implemented by the other executable. When getting or setting grouped parameters, OB-USP-AGENT passes a list of all affected parameters in the group to a single group get or set callback function, improving communication efficiency with the other executable.
+
+Example (for Integrators):
+
+```C
+int VENDOR_Init(void)
+{
+    int err = USP_ERR_OK;
+
+    #define MY_GROUP 1
+    err |= USP_REGISTER_GroupedObject(MY_GROUP, "Device.MyObject.{i}", true);
+
+    err |= USP_REGISTER_GroupedVendorParam_ReadWrite(MY_GROUP, "Device.MyObject.{i}.MyParam", DM_BOOL);
+
+    err |= USP_REGISTER_GroupVendorHooks(MY_GROUP, GetMyParams, SetMyParams, AddMyObject, DelMyObject);
+
+    if (err != USP_ERR_OK)
+    {
+        return USP_ERR_INTERNAL_ERROR;
+    }
+
+    return USP_ERR_OK;
+}
+```
+In the example, the grouped vendor hook callbacks registered by USP_REGISTER_GroupVendorHooks() are called to get, set, add or delete data model parameters and objects associated with MY_GROUP. Providing example implementations of these functions would be too verbose for this guide. Instead, follow the implementation guidelines below.
+
+```C
+int GetMyParams(int group_id, kv_vector_t *params)
+{
+    // params->vector[].key contains parameters to get
+    // Obtain the value of these parameters from the other executable then use USP_ARG_Replace()
+    // or USP_ARG_ReplaceWithHint() to copy the obtained value back into params->vector[]
+    // If some parameters could not be obtained, then just do not call USP_ARG_Replace()
+    // Only return an error if none of the parameters could be obtained (Example: RPC call failure)
+}
+
+int SetMyParams(int group_id, kv_vector_t *params, unsigned *param_types, int *failure_index)
+{
+    // params->vector[].key contains parameters to set
+    // params->vector[].value contains the associated values to set
+    // param_types[] contains the associated type of each parameter (Example: DM_BOOL)
+    // return an error if any of the parameters could not be set.
+    // *failure_index may be used to return the index of the first parameter to fail (index in the params->vector[] and param_types[] arrays).
+    // If this is not known, you should return an index of the value INVALID.
+}
+
+int AddMyObject(int group_id, char *path, int *instance)
+{
+    // return the instance number of the object created by the other executable in *instance
+    // return USP_ERR_CREATION_FAILURE if the object could not be created
+}
+
+int DelMyObject(int group_id, char *path)
+{
+    // return USP_ERR_OBJECT_NOT_DELETABLE if the object could not be deleted
+}
+```
+
+### Refreshing Object Instances
+Changes to object instances are normally signalled using the USP_SIGNAL_ObjectAdded() and 
+USP_SIGNAL_ObjectDeleted() functions. However for some data model objects these events can only be determined by periodically polling the object's instances. This is wasteful to perform continuously, as the object instances are only required when forming a USP response.
+
+Use USP_REGISTER_Object_RefreshInstances() to register an object instance query function which is called on demand.
+
+Example (for Integrators):
+
+```C
+int VENDOR_Init(void)
+{
+    int err = USP_ERR_OK;
+
+    err |= USP_REGISTER_Object("Device.IP.Interface.{i}", NULL, NULL, NULL, NULL, NULL, NULL);
+
+    err |= USP_REGISTER_ObjectRefreshInstances("Device.IP.Interface.{i}", RefreshIPInterfaceInstances);
+
+    if (err != USP_ERR_OK)
+    {
+        return USP_ERR_INTERNAL_ERROR;
+    }
+
+    return USP_ERR_OK;
+}
+
+int RefreshIPInterfaceInstances(int group_id, char *path, int *expiry_period)
+{
+    // Register the currently valid instances for this object and all child objects
+    USP_DM_RefreshInstance("Device.IP.Interface.1");
+    USP_DM_RefreshInstance("Device.IP.Interface.1.IPv4Address.1");
+
+    // cache the object instance numbers for 30 seconds
+    *expiry_period = 30;
+    return USP_ERR_OK;
+}
+
+
+```
+
+

RELEASES.md

@@ -1,4 +1,25 @@
 # Release History
+## Release 3.0.0
+  * MQTT MTP support
+     * MQTT v5.0 and MQTT v3.1.1 support
+     * TLS support
+     * MQTT topic subscriptions
+  * Architectural enhancements
+     * More efficient USP Get and Set for groups of parameters provided by separate components
+     * On-demand querying of table instance numbers
+  * Build support for MUSL libc
+  * Bug fixes
+     * Conformance test failures
+     * Defaulted unique key parameters are validated for uniqueness when adding an instance to a table
+     * Deleting a controller deletes its subscriptions
+     * References to objects without trailing instance numbers do not cause assert
+     * USPRetry set of parameters, should actually be named USPNotifRetry
+  * Other changes
+     * Ignore sender certificate in USP records
+     * Annotations on function prototypes for better Clang static analyzer results
+     * CoAP.Interfaces parameter removed
+     * More robust handling of badly formed percent encoded CoAP resource in URI query option
+
 ## Release 2.1.0
   * Simultaneous sessions with multiple CoAP based controllers are now supported
   * USP Record error handling has been upgraded to be compliant with the USP 1.1 Specification

configure.ac

@@ -12,6 +12,7 @@ LT_PREREQ([2.2])
 LT_INIT([shared disable-static])
 
 # Checks for programs.
+CFLAGS=" -g -O0"
 AC_PROG_CC
 AM_PROG_CC_C_O
 
@@ -20,13 +21,15 @@ AM_PROG_CC_C_O
 PKG_CHECK_MODULES([openssl], [openssl])
 PKG_CHECK_MODULES([sqlite3], [sqlite3])
 PKG_CHECK_MODULES([libcurl], [libcurl])
-PKG_CHECK_MODULES([libcares], [libcares])
 PKG_CHECK_MODULES([zlib], [zlib])
+PKG_CHECK_MODULES([libmosquitto], [libmosquitto])
 
 # Check which flavour of strerror_r is available on the target
 AC_FUNC_STRERROR_R
 
 AC_CHECK_HEADERS([malloc.h])
+AC_CHECK_HEADERS([execinfo.h])
+AC_CHECK_FUNCS([mallinfo])
 
 # Generate Makefiles
 AC_CONFIG_FILES([Makefile])

factory_reset_example.txt

@@ -35,8 +35,8 @@ Device.LocalAgent.Controller.1.Enable "true"
 Device.LocalAgent.Controller.1.AssignedRole "Device.LocalAgent.ControllerTrust.Role.1"
 Device.LocalAgent.Controller.1.PeriodicNotifInterval "86400"
 Device.LocalAgent.Controller.1.PeriodicNotifTime "0001-01-01T00:00:00Z"
-Device.LocalAgent.Controller.1.USPRetryMinimumWaitInterval "5"
-Device.LocalAgent.Controller.1.USPRetryIntervalMultiplier "2000"
+Device.LocalAgent.Controller.1.USPNotifRetryMinimumWaitInterval "5"
+Device.LocalAgent.Controller.1.USPNotifRetryIntervalMultiplier "2000"
 Device.LocalAgent.Controller.1.ControllerCode ""
 Device.LocalAgent.Controller.1.MTP.1.Alias "cpe-1" 
 Device.LocalAgent.Controller.1.MTP.1.Enable "true"