Enable pickle tests

README.md

@@ -422,8 +422,8 @@ distribution packages as listed and rename them as indicated. Copy these to
 | :--------------------- | :----------------- |
 | libyaml.zip   (*node1) | https://github.com/yaml/libyaml/archive/master.zip |
 | boost.tar.gz  (*note3) | https://dl.bintray.com/boostorg/release/1.72.0/source/boost_1_72_0.tar.gz | 
+| googletest.tar.gz      | https://github.com/google/googletest/archive/release-1.8.1.tar.gz |
 | eigen.tar.bz2          | https://gitlab.com/libeigen/eigen/-/archive/3.3.7/eigen-3.3.7.tar.bz2 |
-| googletest.tar.gz      | https://github.com/abseil/googletest/archive/release-1.8.1.tar.gz |
 | mnist.zip     (*note4) | https://github.com/wichtounet/mnist/archive/master.zip |
 | pybind11.tar.gz        | https://github.com/pybind/pybind11/archive/v2.4.2.tar.gz |
 | cereal.tar.gz          | https://github.com/USCiLab/cereal/archive/v1.2.2.tar.gz |

bindings/py/cpp_src/bindings/encoders/py_ScalarEncoder.cpp

@@ -122,5 +122,42 @@ fields are filled in automatically.)");
         self.encode( value, *output );
         return output; },
 R"()");
+
+
+  // pickle
+  py_ScalarEnc.def(py::pickle( 
+  [](const ScalarEncoder& enc) // __getstate__
+  {
+    std::stringstream ss;
+    enc.save(ss);
+
+  /* The values in stringstream are binary so pickle will get confused
+   * trying to treat it as utf8 if you just return ss.str().
+   * So we must treat it as py::bytes.  Some characters could be null values.
+   */
+    return py::bytes( ss.str() );
+  },
+
+  [](py::bytes &s)   // __setstate__
+  {
+  /* pybind11 will pass in the bytes array without conversion.
+   * so we should be able to just create a string to initalize the stringstream.
+   */
+    std::stringstream ss( s.cast<std::string>() );
+    std::unique_ptr<ScalarEncoder> enc(new ScalarEncoder());
+    enc->load(ss);
+
+  /*
+   * The __setstate__ part of the py::pickle() is actually a py::init() with some options.
+   * So the return value can be the object returned by value, by pointer,
+   * or by container (meaning a unique_ptr). SP has a problem with the copy constructor
+   * and pointers have problems knowing who the owner is so lets use unique_ptr.
+   * See: https://pybind11.readthedocs.io/en/stable/advanced/classes.html#custom-constructors
+   */
+    return enc;
+  }));
+
+
+
   }
 }

bindings/py/tests/algorithms/temporal_memory_test.py

@@ -99,7 +99,6 @@ def testNupicTemporalMemoryPickling(self):
                      "Simple NuPIC TemporalMemory pickle/unpickle failed.")
 
 
-  @pytest.mark.skip(reason="Fails with rapidjson internal assertion -- indicates a bad serialization")
   def testNupicTemporalMemorySavingToString(self):
     """Test writing to and reading from TemporalMemory."""
     inputs = SDR( 100 ).randomize( .05 )

bindings/py/tests/encoders/scalar_encoder_test.py

@@ -301,6 +301,27 @@ def testStatistics(self):
         assert( mtr.activationFrequency.max() < 1.75 * .10 )
         assert( mtr.overlap.min() > .85 )
 
-    @pytest.mark.skip(reason="Known issue: https://github.com/htm-community/htm.core/issues/160")
     def testPickle(self):
-        assert(False) # TODO: Unimplemented
+      p = ScalarEncoderParameters()
+      p.size       = 100
+      p.activeBits = 10
+      p.minimum    = 0
+      p.maximum    = 20
+      p.clipInput  = True
+
+      enc = ScalarEncoder( p )
+      import pickle
+
+      picklestr = pickle.dumps(enc)
+      enc2 = pickle.loads(picklestr)
+      #assert enc.parameters == enc2.parameters
+
+      assert enc.size == enc2.size
+
+      out =  SDR( enc.parameters.size )
+      out2 = SDR( enc2.parameters.size )
+      enc.encode(10, out)
+      enc2.encode(10, out2)
+      assert out == out2
+      #TODO add enc == enc2
+

bindings/py/tests/sdr/SDR_test.py

@@ -375,7 +375,6 @@ def testStr(self):
         B.dense = B.dense
         assert(str(B) == "SDR( 100, 100, 1 ) 0, 9999")
 
-    @pytest.mark.skip(reason="Known issue: https://github.com/htm-community/htm.core/issues/160")
     def testPickle(self):
         for sparsity in (0, .3, 1):
             A = SDR((103,))

external/gtest.cmake

@@ -34,7 +34,7 @@
 if(EXISTS "${REPOSITORY_DIR}/build/ThirdParty/share/googletest.tar.gz")
     set(URL "${REPOSITORY_DIR}/build/ThirdParty/share/googletest.tar.gz")
 else()
-    set(URL https://github.com/abseil/googletest/archive/release-1.10.0.tar.gz)
+    set(URL https://github.com/google/googletest/archive/release-1.10.0.tar.gz)
 endif()
 
 #

src/htm/encoders/ScalarEncoder.cpp

@@ -41,7 +41,7 @@ void ScalarEncoder::initialize(const ScalarEncoderParameters &parameters)
   NTA_CHECK( num_active_args != 0u )
       << "Missing argument, need one of: 'activeBits' or 'sparsity'.";
   NTA_CHECK( num_active_args == 1u )
-      << "Too many arguments, choose only one of: 'activeBits' or 'sparsity'.";
+    << "Specified both: 'activeBits' and 'sparsity'. Specify only one of them.";
 
   UInt num_size_args = 0;
   if( parameters.size       > 0u)   { num_size_args++; }
@@ -51,7 +51,7 @@ void ScalarEncoder::initialize(const ScalarEncoderParameters &parameters)
   NTA_CHECK( num_size_args != 0u )
       << "Missing argument, need one of: 'size', 'radius', 'resolution', 'category'.";
   NTA_CHECK( num_size_args == 1u )
-      << "Too many arguments, choose only one of: 'size', 'radius', 'resolution', 'category'.";
+    << "Too many arguments specified: 'size', 'radius', 'resolution', 'category'. Choose only one of them.";
 
   if( parameters.periodic ) {
     NTA_CHECK( not parameters.clipInput )
@@ -79,9 +79,9 @@ void ScalarEncoder::initialize(const ScalarEncoderParameters &parameters)
   if( args_.sparsity > 0.0f ) {
     NTA_CHECK( parameters.sparsity >= 0.0f );
     NTA_CHECK( parameters.sparsity <= 1.0f );
-    NTA_CHECK( args_.size > 0u )
-        << "Argument 'sparsity' requires that the 'size' also be given.";
-    args_.activeBits = (UInt) round( args_.size * args_.sparsity );
+    NTA_CHECK( args_.size > 0u ) << "Argument 'sparsity' requires that the 'size' also be given.";
+    args_.activeBits = static_cast<UInt>(round( args_.size * args_.sparsity ));
+    NTA_CHECK(args_.activeBits > 0) << "sparsity and size must be given so that sparsity * size > 0!";
   }
 
   // Determine resolution & size.
@@ -126,9 +126,11 @@ void ScalarEncoder::initialize(const ScalarEncoderParameters &parameters)
 
   // Determine radius. Always calculate this even if it was given, to correct for rounding error.
   args_.radius = args_.activeBits * args_.resolution;
+  NTA_CHECK(args_.radius > 0);
 
   // Determine sparsity. Always calculate this even if it was given, to correct for rounding error.
-  args_.sparsity = (Real) args_.activeBits / args_.size;
+  args_.sparsity = args_.activeBits / static_cast<Real>(args_.size);
+  NTA_CHECK(args_.sparsity > 0);
 
 
   // Initialize parent class.

src/htm/encoders/ScalarEncoder.hpp

@@ -164,6 +164,8 @@ namespace htm {
       ar(cereal::make_nvp("size", args_.size));
       ar(cereal::make_nvp("radius", args_.radius));
       ar(cereal::make_nvp("resolution", args_.resolution));
+
+      BaseEncoder<Real64>::initialize({ args_.size });
     }
 
     ~ScalarEncoder() override {};

src/test/unit/encoders/ScalarEncoderTest.cpp

@@ -193,7 +193,7 @@ TEST(ScalarEncoder, Serialization) {
     std::stringstream buf;
     enc1->save( buf, JSON );
 
-    //std::cerr << "SERIALIZED:" << std::endl << buf.str() << std::endl;
+    std::cerr << "SERIALIZED:" << std::endl << buf.str() << std::endl;
     buf.seekg(0);
 
     ScalarEncoder enc2;