Integrate alrezni/v2_set_value into master

Source/CNTKv2LibraryDll/API/CNTKLibrary.h

@@ -1704,6 +1704,7 @@ namespace CNTK
 
     protected:
         CNTK_API NDArrayViewPtr Value() const;
+        CNTK_API void SetValue(const NDArrayViewPtr& value);
 
     private:
 #ifdef SWIG
@@ -1998,6 +1999,16 @@ namespace CNTK
             return Variable::Value();
         }
 
+        ///
+        /// Copies the contents of the 'value' NDArrayView into the view backing 'this' 
+        /// parameter's value. The shapes of both views must be identical.
+        ///
+        void SetValue(const NDArrayViewPtr& value)
+        {
+            Variable::SetValue(value);
+            RecordValueUpdate();
+        }
+
         size_t CurrentValueTimeStamp() const { return m_dataFields->m_valueTimeStamp.load(); }
 
         void RecordValueUpdate()

Source/CNTKv2LibraryDll/Variable.cpp

@@ -71,6 +71,37 @@ namespace CNTK
         return m_dataFields->m_value;
     }
 
+    void Variable::SetValue(const NDArrayViewPtr& value)
+    {
+        if (!IsParameter())
+            LogicError("Variable::SetValue can be only invoked on a Parameter variable!");
+        else if (GetDataType() != value->GetDataType()) 
+            LogicError("Variable::SetValue: 'source' and 'destination' have different data types!");
+        else if (Shape() != value->Shape() && (AsTensorShape(Shape()) != AsTensorShape(value->Shape())))
+            LogicError("Variable::SetValue: 'source' and 'destination' have different shapes!");
+
+        bool alreadySet = false;
+        if (m_dataFields->m_initValueFlag)
+        {
+            // In the case of lazy initialization, try to avoid the redundant call to the initializer. 
+            std::call_once(*m_dataFields->m_initValueFlag, [=, &value, &alreadySet] {
+                // If the variable hasn't been initialized yet, clone the content of the supplied value and delete the initializer.
+                m_dataFields->m_value = value->DeepClone(*m_dataFields->m_valueInitializationDevice, false);
+                m_dataFields->m_valueInitializer = nullptr;
+                m_dataFields->m_valueInitializationDevice = nullptr;
+                alreadySet = true;
+            });
+        }
+
+        assert(m_dataFields->m_value != nullptr);
+        if (!alreadySet)
+        {
+            // alreadySet is false, the lambda above wasn't called and the variable has been initialized before,
+            // get a pointer to its value and simply copy the content of the supplied value.
+            m_dataFields->m_value->CopyFrom(*value);
+        }
+    }
+
     static const std::wstring InitializerTypeAttributeName = L"initializerType";
     static const std::wstring OutputRankAttributeName = L"outputRank";
     static const std::wstring FilterRankAttributeName = L"filterRank";

Tests/UnitTests/V2LibraryTests/Common.h

@@ -450,10 +450,9 @@ inline CNTK::FunctionPtr LSTMSequenceClassiferNet(const CNTK::Variable& input, s
     return FullyConnectedLinearLayer(thoughtVectorFunction, numOutputClasses, device, outputName);
 }
 
-template <typename ElementType> 
 inline bool AreEqual(const CNTK::NDArrayViewPtr& view1, const CNTK::NDArrayViewPtr& view2)
 {
-    return AreEqual<ElementType>(*view1, *view2);
+    return CNTK::Internal::AreEqual(*view1, *view2);
 }
 
 inline bool AreEqual(const CNTK::Variable& var1, const CNTK::Variable& var2)

Tests/UnitTests/V2LibraryTests/FunctionTests.cpp

@@ -409,6 +409,87 @@ void TestTimesNodeShapeInference()
     timesNodeShapeInferenceTest(3, 2, 2);
 }
 
+template <typename ElementType>
+void TestChangingParameterValues(size_t rank, const DeviceDescriptor& device)
+{
+    size_t maxDimSize = 15;
+    NDShape shape(rank);
+    for (size_t i = 0; i < rank; ++i)
+        shape[i] = (rand() % maxDimSize) + 1;
+
+    auto numElements = shape.TotalSize();
+
+    auto param = Parameter(shape, AsDataType<ElementType>(), GlorotUniformInitializer(), device);
+    auto plus = Plus(param, param);
+
+
+    std::vector<ElementType> outputData(numElements);
+    ValuePtr outputValue = MakeSharedObject<Value>(MakeSharedObject<NDArrayView>(shape, outputData, false));
+
+    std::unordered_map<Variable, ValuePtr> outputs = { { plus->Output(), outputValue } };
+    plus->Forward({}, outputs, device);
+
+    NDArrayViewPtr cpuView;
+    auto getParameterData = [&cpuView](const Parameter& p) -> const ElementType*
+    {
+          cpuView = (p.Value()->Device() != DeviceDescriptor::CPUDevice()) ?
+                     p.Value()->DeepClone(DeviceDescriptor::CPUDevice()) : p.Value();
+          return cpuView->DataBuffer<ElementType>();
+    };
+
+    auto parameterData = getParameterData(param);
+
+    for (int i = 0; i < numElements; i++)
+    {
+        FloatingPointCompare<ElementType>(outputData[i], 2 * parameterData[i],
+                                          "Function output does not match the expected value.");
+    }
+
+    // Change parameter values element-wise, through a pointer to the writable data buffer.
+    // This only works in CPU mode. In GPU mode the buffer needs to be copied over to a cpuView,
+    // modified there, and then copied back again by calling CopyFrom. The latter is essentially 
+    // what SetValue below does. 
+    if (device == DeviceDescriptor::CPUDevice())
+    {
+        auto data = param.Value()->WritableDataBuffer<ElementType>();
+
+        for (int i = 0; i < numElements; i++)
+        {
+            data[i] *= i;
+        }
+
+        param.RecordValueUpdate();
+        plus->Forward({}, outputs, device);
+        parameterData = getParameterData(param);
+        for (int i = 0; i < numElements; i++)
+        {
+
+            FloatingPointCompare<ElementType>(outputData[i], 2 * parameterData[i],
+                                              "Function output does not match the expected value.");
+        }
+    }
+
+    // Change parameter values directly by calling Parameter::SetValue.
+    std::vector<ElementType> newValues(numElements);
+    for (int i = 0; i < numElements; i++)
+    {
+        newValues[i] = ElementType(1.0) / (i + ElementType(1.0));
+    }
+    auto newValuesNDarray = MakeSharedObject<NDArrayView>(shape, newValues, false);
+
+    param.SetValue(newValuesNDarray);
+    plus->Forward({}, outputs, device);
+    parameterData = getParameterData(param);
+    for (int i = 0; i < numElements; i++)
+    {
+        auto denom = (i + ElementType(1.0));
+        FloatingPointCompare<ElementType>(parameterData[i], ElementType(1.0) / denom, 
+                                          "Parameter valued does not match the expected value.");
+        FloatingPointCompare<ElementType>(outputData[i], ElementType(2.0) / denom, 
+                                          "Function output does not match the expected value.");
+    }
+}
+
 void TestRecurrenceShapeInference()
 {
     auto testShapeInferenceInRecurrence = [](size_t inputRank, size_t outputRank) {
@@ -462,6 +543,16 @@ void FunctionTests()
 {
     fprintf(stderr, "\nFunctionTests..\n");
 
+    TestChangingParameterValues<float>(2, DeviceDescriptor::CPUDevice());
+    if (IsGPUAvailable())
+    {
+        TestChangingParameterValues<double>(3, DeviceDescriptor::GPUDevice(0));
+    } 
+    else
+    {
+        TestChangingParameterValues<double>(3, DeviceDescriptor::CPUDevice());
+    }
+
     TestTimesNodeShapeInference();
     TestRecurrenceShapeInference();
 
@@ -485,3 +576,4 @@ void FunctionTests()
         TestTranspose(3, 1, 2, DeviceDescriptor::GPUDevice(0));
     }
 }
+

Tests/UnitTests/V2LibraryTests/ValueTests.cpp

@@ -238,10 +238,51 @@ void ValueCreationOneHotWithNDMaskTest(const DeviceDescriptor device, bool readO
     CheckValue<ElementType>(testValue, {vocabSize, maxSeqLen, numberOfSequences}, vocabSize, data, seqLenList);
 }
 
+void TestSettingParameterValuesManually(const DeviceDescriptor& device)
+{
+    auto v1_1 = MakeSharedObject<NDArrayView>(0.5, NDShape({ 2, 2 }), device);
+    auto v1_2 = MakeSharedObject<NDArrayView>(0.4, NDShape({ 2, 2 }), device);
+
+    Parameter p1(v1_1);
+    auto value = p1.Value();
+
+    assert(!AreEqual(v1_1, v1_2) && !AreEqual(p1.Value(), v1_2));
+
+    p1.SetValue(v1_2);
+    if (!AreEqual(p1.Value(), v1_2))
+        throw std::runtime_error("Parameter value does match the expected value.");
+
+    Parameter p2(CNTK::NDArrayView::RandomUniform<float>({ 10 }, -0.05, 0.05, 1, device));
+    auto v2 = CNTK::NDArrayView::RandomUniform<float>({ 10 }, -0.05, 0.05, 2, device);
+    assert(!AreEqual(p2.Value(), v2));
+
+    p2.SetValue(v2);
+    if (!AreEqual(p2.Value(), v2))
+        throw std::runtime_error("Parameter value does match the expected value.");
+
+    Parameter p3(NDShape({ 3, 4 }), DataType::Float, GlorotUniformInitializer(), device, L"p3");
+    auto v3 = CNTK::NDArrayView::RandomUniform<float>({ 3, 4 }, -1, 1, 3, device);
+
+    p3.SetValue(v3);
+    if (!AreEqual(p3.Value(), v3))
+        throw std::runtime_error("Parameter value does match the expected value.");
+
+    Parameter p4({ 1 }, DataType::Double, Dictionary(), device, L"p4");
+    auto v4 = MakeSharedObject<NDArrayView>(1.0, NDShape{ 1 }, device);
+
+    // Since p4 initializer is an empty dictionary, lazy-initialization (triggered by the value getter: p4.Value())
+    // should fail. However, the setter will override the bogus initializer and init p4 by copying v4 content.
+    p4.SetValue(v4);
+    if (!AreEqual(p4.Value(), v4))
+        throw std::runtime_error("Parameter value does match the expected value.");
+}
+
 void ValueTests()
 {
     fprintf(stderr, "\nValueTests..\n");
 
+    TestSettingParameterValuesManually(DeviceDescriptor::CPUDevice());
+
     ValueCreationNoNDMaskTest<float>(DeviceDescriptor::CPUDevice(), false);
     ValueCreationNoNDMaskTest<double>(DeviceDescriptor::CPUDevice(), true);
     ValueCreationWithNDMaskTest<double>(DeviceDescriptor::CPUDevice(), false);
@@ -250,8 +291,11 @@ void ValueTests()
     ValueCreationOneHotNoNDMaskTest<double>(DeviceDescriptor::CPUDevice(), true);
     ValueCreationOneHotWithNDMaskTest<double>(DeviceDescriptor::CPUDevice(), false);
     ValueCreationOneHotWithNDMaskTest<float>(DeviceDescriptor::CPUDevice(), true);
+
     if (IsGPUAvailable())
     {
+        TestSettingParameterValuesManually(DeviceDescriptor::GPUDevice(0));
+
         ValueCreationNoNDMaskTest<double>(DeviceDescriptor::GPUDevice(0), false);
         ValueCreationNoNDMaskTest<float>(DeviceDescriptor::GPUDevice(0), true);
         ValueCreationWithNDMaskTest<float>(DeviceDescriptor::GPUDevice(0), false);

bindings/python/cntk/ops/tests/variables_test.py

@@ -9,7 +9,7 @@
 """
 
 from ..variables import *
-from .. import times, placeholder_variable, constant
+from .. import times, placeholder_variable, constant, plus
 import numpy as np
 
 import pytest
@@ -42,6 +42,22 @@ def test_variable_shape(variable_type, shape):
         [[[1,2],[3,4],[5,6]],[[1,2],[3,4],[5,6]]]
         ]
 
+def test_parameter_set_value():
+    p = Parameter(shape=(2,3), init=1);
+    n = np.random.randn(2, 3)
+    p.value = n
+    assert np.all(p.value == n.astype(p.dtype))
+
+    n = np.reshape(np.arange(6), (2, 3))
+    p.value = n
+    op = plus(p, p)
+    state, output = op.forward({}, op.outputs, op.outputs)
+    value = output[op.output]
+    assert np.all(value == 2*n.astype(p.dtype))
+
+    p.value = sanitize_value(p.shape, 1.0, np.float32, None)
+    assert np.all(p.value == np.ones((2,3)))
+
 @pytest.mark.parametrize("value", VALUES)
 def test_constant_value(value):
     c = Constant(value=value)

bindings/python/cntk/ops/variables.py

@@ -1,7 +1,7 @@
 import numpy as np
 from cntk import cntk_py, utils
 from ..tensor import TensorOpsMixin
-from ..utils import typemap, sanitize_precision, sanitize_value, sanitize_dtype_cntk
+from ..utils import typemap, sanitize_precision, sanitize_value, sanitize_dtype_cntk, _create_NDArrayView_from_NumPy
 
 class VariableMixin:
     '''
@@ -178,6 +178,17 @@ def value(self):
         '''
         return super(Parameter, self).value().to_numpy()
 
+    @value.setter
+    def value(self, val):
+        if isinstance(val, np.ndarray):
+            ndarray = _create_NDArrayView_from_NumPy(val.astype(self.dtype))
+            super().set_value(ndarray)
+        elif isinstance(val, cntk_py.NDArrayView):
+            super().set_value(val)
+        else:
+            raise TypeError("Unsupported value type: %s", type(val))
+
+
 class Constant(VariableMixin, TensorOpsMixin, cntk_py.Constant):
     '''
     A constant value. It can be a scalar, vector, matrix, or tensor