Merge branch 'master' into feature/forecast-enhancements-part-2

Author: tveasey

3rd_party/3rd_party.sh

@@ -63,22 +63,22 @@ case `uname` in
         if [ -z "$CPP_CROSS_COMPILE" ] ; then
             ldd --version 2>&1 | grep musl > /dev/null
             if [ $? -ne 0 ] ; then
-                APR_LOCATION=/usr/local/gcc62/lib
+                APR_LOCATION=/usr/local/gcc73/lib
                 APR_EXTENSION=1.so.0
-                BOOST_LOCATION=/usr/local/gcc62/lib
+                BOOST_LOCATION=/usr/local/gcc73/lib
                 BOOST_COMPILER=gcc
                 BOOST_EXTENSION=mt-1_65_1.so.1.65.1
                 BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
-                LOG4CXX_LOCATION=/usr/local/gcc62/lib
+                LOG4CXX_LOCATION=/usr/local/gcc73/lib
                 LOG4CXX_EXTENSION=.so.10
-                XML_LOCATION=/usr/local/gcc62/lib
+                XML_LOCATION=/usr/local/gcc73/lib
                 XML_EXTENSION=.so.2
                 # Ship the version of expat that came with the apr-util library
-                EXPAT_LOCATION=/usr/local/gcc62/lib
+                EXPAT_LOCATION=/usr/local/gcc73/lib
                 EXPAT_EXTENSION=.so.0
-                GCC_RT_LOCATION=/usr/local/gcc62/lib64
+                GCC_RT_LOCATION=/usr/local/gcc73/lib64
                 GCC_RT_EXTENSION=.so.1
-                STL_LOCATION=/usr/local/gcc62/lib64
+                STL_LOCATION=/usr/local/gcc73/lib64
                 STL_PREFIX=libstdc++
                 STL_EXTENSION=.so.6
                 ATOMIC_LOCATION=

bin/autodetect/Main.cc

@@ -45,6 +45,7 @@
 #include <api/CIoManager.h>
 #include <api/CJsonOutputWriter.h>
 #include <api/CLengthEncodedInputParser.h>
+#include <api/CModelSnapshotJsonWriter.h>
 #include <api/COutputChainer.h>
 #include <api/CSingleStreamDataAdder.h>
 #include <api/CSingleStreamSearcher.h>
@@ -271,8 +272,7 @@ int main(int argc, char **argv)
 
     ml::core::CJsonOutputStreamWrapper wrappedOutputStream(ioMgr.outputStream());
 
-    // output writer for CFieldDataTyper and persistence callback
-    ml::api::CJsonOutputWriter outputWriter(jobId, wrappedOutputStream);
+    ml::api::CModelSnapshotJsonWriter modelSnapshotWriter(jobId, wrappedOutputStream);
     if (fieldConfig.initFromCmdLine(fieldConfigFile,
                                     clauseTokens) == false)
     {
@@ -286,8 +286,7 @@ int main(int argc, char **argv)
                              fieldConfig,
                              modelConfig,
                              wrappedOutputStream,
-                             boost::bind(&ml::api::CJsonOutputWriter::reportPersistComplete,
-                                         &outputWriter, _1, _2, _3, _4, _5, _6, _7, _8),
+                             boost::bind(&ml::api::CModelSnapshotJsonWriter::write, &modelSnapshotWriter, _1),
                              periodicPersister.get(),
                              maxQuantileInterval,
                              timeField,
@@ -312,8 +311,10 @@ int main(int argc, char **argv)
     // Chain the categorizer's output to the anomaly detector's input
     ml::api::COutputChainer outputChainer(job);
 
+    ml::api::CJsonOutputWriter fieldDataTyperOutputWriter(jobId, wrappedOutputStream);
+
     // The typer knows how to assign categories to records
-    ml::api::CFieldDataTyper typer(jobId, fieldConfig, limits, outputChainer, outputWriter);
+    ml::api::CFieldDataTyper typer(jobId, fieldConfig, limits, outputChainer, fieldDataTyperOutputWriter);
 
     if (fieldConfig.fieldNameSuperset().count(ml::api::CFieldDataTyper::MLCATEGORY_NAME) > 0)
     {
@@ -339,7 +340,7 @@ int main(int argc, char **argv)
     // as it must be finalised before the skeleton is destroyed, and C++
     // destruction order means the skeleton will be destroyed before the output
     // writer as it was constructed last.
-    outputWriter.finalise();
+    fieldDataTyperOutputWriter.finalise();
 
     if (!ioLoopSucceeded)
     {

build-setup/linux.md

@@ -6,16 +6,16 @@ You will need the following environment variables to be defined:
 
 - `JAVA_HOME` - Should point to the JDK you want to use to run Gradle.
 - `CPP_SRC_HOME` - Only required if building the C++ code directly using `make`, as Gradle sets it automatically.
-- `PATH` - Must have `/usr/local/gcc62/bin` before `/usr/bin` and `/bin`.
-- `LD_LIBRARY_PATH` - Must have `/usr/local/gcc62/lib64` and `/usr/local/gcc62/lib` before `/usr/lib` and `/lib`.
+- `PATH` - Must have `/usr/local/gcc73/bin` before `/usr/bin` and `/bin`.
+- `LD_LIBRARY_PATH` - Must have `/usr/local/gcc73/lib64` and `/usr/local/gcc73/lib` before `/usr/lib` and `/lib`.
 
 For example, you might create a `.bashrc` file in your home directory containing something like this:
 
 ```
 umask 0002
 export JAVA_HOME=/usr/local/jdk1.8.0_121
-export LD_LIBRARY_PATH=/usr/local/gcc62/lib64:/usr/local/gcc62/lib:/usr/lib:/lib
-export PATH=$JAVA_HOME/bin:/usr/local/gcc62/bin:/usr/bin:/bin:/usr/sbin:/sbin:/home/vagrant/bin
+export LD_LIBRARY_PATH=/usr/local/gcc73/lib64:/usr/local/gcc73/lib:/usr/lib:/lib
+export PATH=$JAVA_HOME/bin:/usr/local/gcc73/bin:/usr/bin:/bin:/usr/sbin:/sbin:/home/vagrant/bin
 # Only required if building the C++ code directly using make - adjust depending on the location of your Git clone
 export CPP_SRC_HOME=$HOME/ml-cpp
 ```
@@ -38,7 +38,7 @@ On other Linux distributions the package names are generally the same and you ju
 Most of the tools are built via a GNU "configure" script. There are some environment variables that affect the behaviour of this. Therefore, when building ANY tool on Linux, set the following environment variable:
 
 ```
-export CXX='g++ -std=gnu++0x'
+export CXX='g++ -std=gnu++14'
 unset LIBRARY_PATH
 ```
 
@@ -48,22 +48,47 @@ The `CXX` environment variable only needs to be set when building tools on Linux
 
 We have to build on old Linux versions to enable our software to run on the older versions of Linux that users have.  However, this means the default compiler on our Linux build servers is also very old.  To enable use of more modern C++ features, we use the default compiler to build a newer version of gcc and then use that to build all our other dependencies.
 
-Download `gcc-6.2.0.tar.bz2` from <http://ftpmirror.gnu.org/gcc/gcc-6.2.0/gcc-6.2.0.tar.bz2>.
+Download `gcc-7.3.0.tar.gz` from <http://ftpmirror.gnu.org/gcc/gcc-7.3.0/gcc-7.3.0.tar.gz>.
 
 Unlike most automake-based tools, gcc must be built in a directory adjacent to the directory containing its source code, so build and install it like this:
 
 ```
-tar jxvf gcc-6.2.0.tar.bz2
-cd gcc-6.2.0
+tar zxvf gcc-7.3.0.tar.gz
+cd gcc-7.3.0
 contrib/download_prerequisites
 cd ..
-mkdir gcc-6.2.0-build
-cd gcc-6.2.0-build
-../gcc-6.2.0/configure --prefix=/usr/local/gcc62 --enable-languages=c,c++ --with-system-zlib --disable-multilib
+mkdir gcc-7.3.0-build
+cd gcc-7.3.0-build
+unset CXX
+unset LD_LIBRARY_PATH
+export PATH=/usr/bin:/bin:/usr/sbin:/sbin
+../gcc-7.3.0/configure --prefix=/usr/local/gcc73 --enable-languages=c,c++ --enable-vtable-verify --with-system-zlib --disable-multilib
 make -j 6
 sudo make install
 ```
 
+It's important that gcc itself is built using the system compiler in C++98 mode, hence the adjustment to `PATH` and unsetting of `CXX` and `LD_LIBRARY_PATH`.
+
+After the gcc build is complete, if you are going to work through the rest of these instructions in the same shell remember to reset the `CXX` environment variable so that the remaining C++ components get built with C++14:
+
+```
+export CXX='g++ -std=gnu++14'
+```
+
+To confirm that everything works correctly run:
+
+```
+g++ --version
+```
+
+It should print:
+
+```
+g++ (GCC) 7.3.0
+```
+
+in the first line of the output. If it doesn't then double check that `/usr/local/gcc73/bin` is near the beginning of your `PATH`.
+
 ### Git
 
 Modern versions of Linux will come with Git in their package repositories, and (since we're not redistributing it so don't really care about the exact version used) this is the easiest way to install it. The command will be:
@@ -72,7 +97,7 @@ Modern versions of Linux will come with Git in their package repositories, and (
 sudo yum install git
 ```
 
-on RHEL clones, or for SLES use YaST. **However**, Jenkins requires at minimum version 1.7.9 of Git, so if the version that yum installs is older you'll still have to build it from scratch. In this case, you may need to uninstall the version that yum installed:
+on RHEL clones. **However**, Jenkins requires at minimum version 1.7.9 of Git, so if the version that yum installs is older you'll still have to build it from scratch. In this case, you may need to uninstall the version that yum installed:
 
 ```
 git --version
@@ -107,7 +132,7 @@ Anonymous FTP to ftp.xmlsoft.org, change directory to libxml2, switch to binary
 Uncompress and untar the resulting file. Then run:
 
 ```
-./configure --prefix=/usr/local/gcc62 --without-python --without-readline
+./configure --prefix=/usr/local/gcc73 --without-python --without-readline
 ```
 
 This should build an appropriate Makefile. Assuming it does, type:
@@ -121,36 +146,36 @@ to install.
 
 ### APR
 
-For Linux, before building log4cxx you must download the Apache Portable Runtime (APR), and its utilities from <http://mirrors.sonic.net/apache/apr/apr-1.5.2.tar.bz2>.
+For Linux, before building log4cxx you must download the Apache Portable Runtime (APR) from <http://archive.apache.org/dist/apr/apr-1.5.2.tar.bz2>.
 
 Extract the tarball to a temporary directory:
 
 ```
-tar xvf apr-1.5.2.tar
+tar jxvf apr-1.5.2.tar.bz2
 ```
 
 Build using:
 
 ```
-./configure --prefix=/usr/local/gcc62
+./configure --prefix=/usr/local/gcc73
 make
 sudo make install
 ```
 
 ### APR utilities
 
-For Linux, before building log4cxx you must download the Apache Portable Runtime (APR) utilities from <http://mirrors.sonic.net/apache/apr/apr-util-1.5.4.tar.bz2>.
+For Linux, before building log4cxx you must download the Apache Portable Runtime (APR) utilities from <http://archive.apache.org/dist/apr/apr-util-1.5.4.tar.bz2>.
 
 Extract the tarball to a temporary directory:
 
 ```
-tar xvf apr-util-1.5.4.tar
+tar jxvf apr-util-1.5.4.tar.bz2
 ```
 
 Build using:
 
 ```
-./configure --prefix=/usr/local/gcc62 --with-apr=/usr/local/gcc62/bin/apr-1-config --with-expat=builtin
+./configure --prefix=/usr/local/gcc73 --with-apr=/usr/local/gcc73/bin/apr-1-config --with-expat=builtin
 make
 sudo make install
 ```
@@ -179,7 +204,7 @@ to:
 
 ```
 #if LOG4CXX_HELGRIND
-#define _LOG4CXX_OBJECTPTR_INIT(x) : ObjectPtrBase() { exchange(x); 
+#define _LOG4CXX_OBJECTPTR_INIT(x) : ObjectPtrBase() { exchange(x);
 #else
 #define _LOG4CXX_OBJECTPTR_INIT(x) : ObjectPtrBase(), p(x) {
 #endif
@@ -239,7 +264,7 @@ In `src/test/cpp/xml/domtestcase.cpp` replace `0x` with `(char)0x` on lines 193
 Once all the changes are made, configure using:
 
 ```
-./configure --prefix=/usr/local/gcc62 --with-charset=utf-8 --with-logchar=utf-8 --with-apr=/usr/local/gcc62 --with-apr-util=/usr/local/gcc62
+./configure --prefix=/usr/local/gcc73 --with-charset=utf-8 --with-logchar=utf-8 --with-apr=/usr/local/gcc73 --with-apr-util=/usr/local/gcc73
 ```
 
 This should build an appropriate Makefile. Assuming it does, type:
@@ -264,12 +289,12 @@ bzip2 -cd boost_1_65_1.tar.bz2 | tar xvf -
 In the resulting `boost_1_65_1` directory, run:
 
 ```
-./bootstrap.sh cxxflags=-std=gnu++0x --without-libraries=context --without-libraries=coroutine --without-libraries=graph_parallel --without-libraries=log --without-libraries=mpi --without-libraries=python --without-libraries=test --without-icu
+./bootstrap.sh cxxflags=-std=gnu++14 --without-libraries=context --without-libraries=coroutine --without-libraries=graph_parallel --without-libraries=log --without-libraries=mpi --without-libraries=python --without-libraries=test --without-icu
 ```
 
 This should build the `b2` program, which in turn is used to build Boost.
 
-Edit boost/unordered/detail/implementation.hpp and change line 270 from:
+Edit `boost/unordered/detail/implementation.hpp` and change line 270 from:
 
 ```
     (17ul)(29ul)(37ul)(53ul)(67ul)(79ul) \
@@ -297,7 +322,7 @@ Finally, run:
 
 ```
 ./b2 -j6 --layout=versioned --disable-icu pch=off optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS
-sudo env PATH="$PATH" ./b2 install --prefix=/usr/local/gcc62 --layout=versioned --disable-icu pch=off optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS
+sudo env PATH="$PATH" ./b2 install --prefix=/usr/local/gcc73 --layout=versioned --disable-icu pch=off optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS
 ```
 
 to install the Boost headers and libraries.  (Note the `env PATH="$PATH"` bit in the install command - this is because `sudo` usually resets `PATH` and that will cause Boost to rebuild everything again with the default compiler as part of the install!)
@@ -309,7 +334,7 @@ Download the latest version of cppunit from <http://dev-www.libreoffice.org/src/
 Untar it to a temporary directory and run:
 
 ```
-./configure --prefix=/usr/local/gcc62
+./configure --prefix=/usr/local/gcc73
 ```
 
 This should build an appropriate Makefile. Assuming it does, type:
@@ -334,7 +359,7 @@ bzip2 -cd patchelf-0.9.tar.bz2 | tar xvf -
 In the resulting `patchelf-0.9` directory, run the:
 
 ```
-./configure --prefix=/usr/local/gcc62
+./configure --prefix=/usr/local/gcc73
 ```
 
 script. This should build an appropriate Makefile. Assuming it does, run:

build-setup/windows.md

@@ -161,9 +161,9 @@ nmake install
 
 ### APR
 
-Download the Windows source for version 1.5.2 of the Apache Portable Runtime (APR) from <http://apr.apache.org/download.cgi>.
+Download the Windows source for version 1.5.2 of the Apache Portable Runtime (APR) from <http://archive.apache.org/dist/apr/apr-1.5.2-win32-src.zip>.
 
-The file should be called `apr-1.5.2-win32-src.zip`. Unzip the file into `C:\tools`. You should end up with a subdirectory called `apr-1.5.2`. Rename this to simply `apr`.
+Unzip the file into `C:\tools`. You should end up with a subdirectory called `apr-1.5.2`. Rename this to simply `apr`.
 
 Within the files `C:\tools\apr\apr.mak` and `C:\tools\apr\libapr.mak`, globally replace:
 
@@ -189,9 +189,9 @@ nmake -f Makefile.win PREFIX=C:\usr\local ARCH="x64 Release" install
 
 ### APR iconv
 
-Download the Windows source for version 1.2.1 of the Apache Portable Runtime (APR) version of iconv from <http://apr.apache.org/download.cgi>.
+Download the Windows source for version 1.2.1 of the Apache Portable Runtime (APR) version of iconv from <http://archive.apache.org/dist/apr/apr-iconv-1.2.1-win32-src.zip>.
 
-The file should be called `apr-iconv-1.2.1-win32-src.zip`. Unzip the file into `C:\tools`. You should end up with a subdirectory called `apr-iconv-1.2.1`. Rename this to simply `apr-iconv`.
+Unzip the file into `C:\tools`. You should end up with a subdirectory called `apr-iconv-1.2.1`. Rename this to simply `apr-iconv`.
 
 Within the files `C:\tools\apr-iconv\apriconv.mak` and `C:\tools\apr-iconv\libapriconv.mak`, globally replace:
 
@@ -229,9 +229,9 @@ Note that this does not install all the iconv utilities - this will be done when
 
 ### APR util
 
-Download the Windows source for version 1.5.4 of the Apache Portable Runtime (APR) utilities from <http://apr.apache.org/download.cgi> .
+Download the Windows source for version 1.5.4 of the Apache Portable Runtime (APR) utilities from <http://archive.apache.org/dist/apr/apr-util-1.5.4-win32-src.zip>.
 
-The file should be called `apr-util-1.5.4-win32-src.zip`. Unzip the file into `C:\tools`. You should end up with a subdirectory called `apr-util-1.5.4`. Rename this to simply `apr-util`.
+Unzip the file into `C:\tools`. You should end up with a subdirectory called `apr-util-1.5.4`. Rename this to simply `apr-util`.
 
 Within the files `C:\tools\apr-util\aprutil.mak` and `C:\tools\apr-util\libaprutil.mak`, globally replace: