[ML] Wire in change detection/modelling to our univariate time series model (#11)

This wires in change detection and starts some unit testing of CTimeSeriesModel with change 
points. There is some more work to be done to avoid using level shifts to try and fit other types 
of change points, such as scaling.

Author: tveasey

include/maths/CBasicStatistics.h

@@ -221,6 +221,16 @@ class MATHS_EXPORT CBasicStatistics
                 }
             }
 
+            //! Update the moments with the collection \p x.
+            template<typename U, std::size_t N>
+            void add(const core::CSmallVector<U, N> &x)
+            {
+                for (const auto &xi : x)
+                {
+                    this->add(xi);
+                }
+            }
+
             //! Update the moments with the collection \p x.
             template<typename U>
             void add(const std::vector<SSampleCentralMoments<U, ORDER>> &x)

include/maths/CModel.h

@@ -49,9 +49,11 @@ class MATHS_EXPORT CModelParams
 {
     public:
         CModelParams(core_t::TTime bucketLength,
-                     const double &learnRate,
-                     const double &decayRate,
-                     double minimumSeasonalVarianceScale);
+                     double learnRate,
+                     double decayRate,
+                     double minimumSeasonalVarianceScale,
+                     core_t::TTime minimumTimeToDetectChange,
+                     core_t::TTime maximumTimeToTestForChange);
 
         //! Get the bucket length.
         core_t::TTime bucketLength(void) const;
@@ -68,6 +70,15 @@ class MATHS_EXPORT CModelParams
         //! Get the minimum seasonal variance scale.
         double minimumSeasonalVarianceScale(void) const;
 
+        //! Check if we should start testing for a change point in the model.
+        bool testForChange(core_t::TTime changeInterval) const;
+
+        //! Get the minimum time to detect a change point in the model.
+        core_t::TTime minimumTimeToDetectChange(core_t::TTime timeSinceLastChangePoint) const;
+
+        //! Get the maximum time to test for a change point in the model.
+        core_t::TTime maximumTimeToTestForChange(void) const;
+
         //! Set the probability that the bucket will be empty for the model.
         void probabilityBucketEmpty(double probability);
 
@@ -83,6 +94,10 @@ class MATHS_EXPORT CModelParams
         double m_DecayRate;
         //! The minimum seasonal variance scale.
         double m_MinimumSeasonalVarianceScale;
+        //! The minimum time permitted to detect a change in the model.
+        core_t::TTime m_MinimumTimeToDetectChange;
+        //! The maximum time permitted to test for a change in the model.
+        core_t::TTime m_MaximumTimeToTestForChange;
         //! The probability that a bucket will be empty for the model.
         double m_ProbabilityBucketEmpty;
 };
@@ -97,8 +112,6 @@ class MATHS_EXPORT CModelAddSamplesParams
 
     public:
         CModelAddSamplesParams(void);
-        CModelAddSamplesParams(const CModelAddSamplesParams &) = delete;
-        const CModelAddSamplesParams &operator=(const CModelAddSamplesParams &) = delete;
 
         //! Set whether or not the data are integer valued.
         CModelAddSamplesParams &integer(bool integer);
@@ -160,8 +173,6 @@ class MATHS_EXPORT CModelProbabilityParams
 
     public:
         CModelProbabilityParams(void);
-        CModelProbabilityParams(const CModelAddSamplesParams &) = delete;
-        const CModelProbabilityParams &operator=(const CModelAddSamplesParams &) = delete;
 
         //! Set the tag for the entity for which to compute the probability.
         CModelProbabilityParams &tag(std::size_t tag);
@@ -278,6 +289,9 @@ class MATHS_EXPORT CModel
             E_Reset    //!< Model reset.
         };
 
+        //! Combine the results \p lhs and \p rhs.
+        static EUpdateResult combine(EUpdateResult lhs, EUpdateResult rhs);
+
     public:
         CModel(const CModelParams &params);
         virtual ~CModel(void) = default;

include/maths/CNaiveBayes.h

@@ -11,9 +11,11 @@
 
 #include <maths/CPrior.h>
 
+#include <boost/optional.hpp>
 #include <boost/unordered_map.hpp>
 
 #include <cstddef>
+#include <string>
 #include <vector>
 
 namespace ml
@@ -49,12 +51,18 @@ class MATHS_EXPORT CNaiveBayesFeatureDensity
         //! Persist state by passing information to \p inserter.
         virtual void acceptPersistInserter(core::CStatePersistInserter &inserter) const = 0;
 
+        //! Set the data type.
+        virtual void dataType(maths_t::EDataType dataType) = 0;
+
         //! Add the value \p x.
         virtual void add(const TDouble1Vec &x) = 0;
 
         //! Compute the log value of the density function at \p x.
         virtual double logValue(const TDouble1Vec &x) const = 0;
 
+        //! Compute the density at the mode.
+        virtual double logMaximumValue() const = 0;
+
         //! Age out old values density to account for \p time passing.
         virtual void propagateForwardsByTime(double time) = 0;
 
@@ -69,6 +77,9 @@ class MATHS_EXPORT CNaiveBayesFeatureDensity
 
         //! Get a checksum for this object.
         virtual uint64_t checksum(uint64_t seed) const = 0;
+
+        //! Get a human readable description of the class density function.
+        virtual std::string print() const = 0;
 };
 
 //! \brief An implementation of the class conditional density function
@@ -77,7 +88,7 @@ class MATHS_EXPORT CNaiveBayesFeatureDensityFromPrior final : public CNaiveBayes
 {
     public:
         CNaiveBayesFeatureDensityFromPrior() = default;
-        CNaiveBayesFeatureDensityFromPrior(CPrior &prior);
+        CNaiveBayesFeatureDensityFromPrior(const CPrior &prior);
 
         //! Create and return a clone.
         //!
@@ -97,6 +108,12 @@ class MATHS_EXPORT CNaiveBayesFeatureDensityFromPrior final : public CNaiveBayes
         //! Compute the log value of the density function at \p x.
         virtual double logValue(const TDouble1Vec &x) const;
 
+        //! Compute the density at the mode.
+        virtual double logMaximumValue() const;
+
+        //! Set the data type.
+        virtual void dataType(maths_t::EDataType dataType);
+
         //! Age out old values density to account for \p time passing.
         virtual void propagateForwardsByTime(double time);
 
@@ -112,6 +129,9 @@ class MATHS_EXPORT CNaiveBayesFeatureDensityFromPrior final : public CNaiveBayes
         //! Get a checksum for this object.
         virtual uint64_t checksum(uint64_t seed) const;
 
+        //! Get a human readable description of the class density function.
+        virtual std::string print() const;
+
     private:
         using TPriorPtr = boost::shared_ptr<CPrior>;
 
@@ -128,16 +148,24 @@ class MATHS_EXPORT CNaiveBayes
         using TDoubleSizePrVec = std::vector<TDoubleSizePr>;
         using TDouble1Vec = core::CSmallVector<double, 1>;
         using TDouble1VecVec = std::vector<TDouble1Vec>;
+        using TOptionalDouble = boost::optional<double>;
 
     public:
         explicit CNaiveBayes(const CNaiveBayesFeatureDensity &exemplar,
-                             double decayRate = 0.0);
+                             double decayRate = 0.0,
+                             TOptionalDouble minMaxLogLikelihoodToUseFeature = TOptionalDouble());
         CNaiveBayes(const SDistributionRestoreParams &params,
                     core::CStateRestoreTraverser &traverser);
 
         //! Persist state by passing information to \p inserter.
         void acceptPersistInserter(core::CStatePersistInserter &inserter) const;
 
+        //! Efficiently swap the contents of this and \p other.
+        void swap(CNaiveBayes &other);
+
+        //! Check if any training data has been added initialized.
+        bool initialized() const;
+
         //! This can be used to optionally seed the class counts
         //! with \p counts. These are added on to data class counts
         //! to compute the class posterior probabilities.
@@ -153,11 +181,14 @@ class MATHS_EXPORT CNaiveBayes
         //! for that feature.
         void addTrainingDataPoint(std::size_t label, const TDouble1VecVec &x);
 
+        //! Set the data type.
+        void dataType(maths_t::EDataType dataType);
+
         //! Age out old values from the class conditional densities
         //! to account for \p time passing.
         void propagateForwardsByTime(double time);
 
-        //! Get the top \p n class probabilities for \p features.
+        //! Get the top \p n class probabilities for \p x.
         //!
         //! \param[in] n The number of class probabilities to estimate.
         //! \param[in] x The feature values.
@@ -167,6 +198,23 @@ class MATHS_EXPORT CNaiveBayes
         TDoubleSizePrVec highestClassProbabilities(std::size_t n,
                                                    const TDouble1VecVec &x) const;
 
+        //! Get the probability of the class labeled \p label for \p x.
+        //!
+        //! \param[in] label The label of the class of interest.
+        //! \param[in] x The feature values.
+        //! \note \p x size should be equal to the number of features.
+        //! A feature is missing is indicated by passing an empty vector
+        //! for that feature.
+        double classProbability(std::size_t label, const TDouble1VecVec &x) const;
+
+        //! Get the probabilities of all the classes for \p x.
+        //!
+        //! \param[in] x The feature values.
+        //! \note \p x size should be equal to the number of features.
+        //! A feature is missing is indicated by passing an empty vector
+        //! for that feature.
+        TDoubleSizePrVec classProbabilities(const TDouble1VecVec &x) const;
+
         //! Debug the memory used by this object.
         void debugMemoryUsage(core::CMemoryUsage::TMemoryUsagePtr mem) const;
 
@@ -176,6 +224,9 @@ class MATHS_EXPORT CNaiveBayes
         //! Get a checksum for this object.
         uint64_t checksum(uint64_t seed = 0) const;
 
+        //! Get a human readable description of the classifier.
+        std::string print() const;
+
     private:
         using TFeatureDensityPtr = boost::shared_ptr<CNaiveBayesFeatureDensity>;
         using TFeatureDensityPtrVec = std::vector<TFeatureDensityPtr>;
@@ -212,6 +263,13 @@ class MATHS_EXPORT CNaiveBayes
         bool validate(const TDouble1VecVec &x) const;
 
     private:
+        //! It is not always appropriate to use features with very low
+        //! probability in all classes to discriminate: the class choice
+        //! will be very sensitive to the underlying conditional density
+        //! model. This is a cutoff (for the minimum maximum class log
+        //! likelihood) in order to use a feature.
+        TOptionalDouble m_MinMaxLogLikelihoodToUseFeature;
+
         //! Controls the rate at which data are aged out.
         double m_DecayRate;
 

include/maths/CRestoreParams.h

@@ -9,6 +9,7 @@
 
 #include <core/CoreTypes.h>
 
+#include <maths/Constants.h>
 #include <maths/ImportExport.h>
 #include <maths/MathsTypes.h>
 
@@ -20,33 +21,15 @@ namespace maths
 {
 class CModelParams;
 
-//! \brief Gatherers up extra parameters supplied when restoring
-//! time series decompositions.
-struct MATHS_EXPORT STimeSeriesDecompositionRestoreParams
-{
-    STimeSeriesDecompositionRestoreParams(double decayRate,
-                                          core_t::TTime minimumBucketLength,
-                                          std::size_t componentSize);
-
-    //! The rate at which decomposition loses information.
-    double s_DecayRate;
-
-    //! The data bucket length.
-    core_t::TTime s_MinimumBucketLength;
-
-    //! The decomposition seasonal component size.
-    std::size_t s_ComponentSize;
-};
-
 //! \brief Gatherers up extra parameters supplied when restoring
 //! distribution models.
 struct MATHS_EXPORT SDistributionRestoreParams
 {
     SDistributionRestoreParams(maths_t::EDataType dataType,
                                double decayRate,
-                               double minimumClusterFraction,
-                               double minimumClusterCount,
-                               double minimumCategoryCount);
+                               double minimumClusterFraction = MINIMUM_CLUSTER_SPLIT_FRACTION,
+                               double minimumClusterCount = MINIMUM_CLUSTER_SPLIT_COUNT,
+                               double minimumCategoryCount = MINIMUM_CATEGORY_COUNT);
 
     //! The type of data being clustered.
     maths_t::EDataType s_DataType;
@@ -66,6 +49,31 @@ struct MATHS_EXPORT SDistributionRestoreParams
 
 //! \brief Gatherers up extra parameters supplied when restoring
 //! time series decompositions.
+struct MATHS_EXPORT STimeSeriesDecompositionRestoreParams
+{
+    STimeSeriesDecompositionRestoreParams(double decayRate,
+                                          core_t::TTime minimumBucketLength,
+                                          std::size_t componentSize,
+                                          const SDistributionRestoreParams &changeModelParams);
+    STimeSeriesDecompositionRestoreParams(double decayRate,
+                                          core_t::TTime minimumBucketLength,
+                                          const SDistributionRestoreParams &changeModelParams);
+
+    //! The rate at which decomposition loses information.
+    double s_DecayRate;
+
+    //! The data bucket length.
+    core_t::TTime s_MinimumBucketLength;
+
+    //! The decomposition seasonal component size.
+    std::size_t s_ComponentSize;
+
+    //! The change model distributions' restore parameters.
+    SDistributionRestoreParams s_ChangeModelParams;
+};
+
+//! \brief Gatherers up extra parameters supplied when restoring
+//! time series models.
 struct MATHS_EXPORT SModelRestoreParams
 {
     using TModelParamsCRef = boost::reference_wrapper<const CModelParams>;
@@ -80,7 +88,7 @@ struct MATHS_EXPORT SModelRestoreParams
     //! The time series decomposition restore parameters.
     STimeSeriesDecompositionRestoreParams s_DecompositionParams;
 
-    //! The time series decomposition restore parameters.
+    //! The time series residual distribution restore parameters.
     SDistributionRestoreParams s_DistributionParams;
 };