From de7616e1f567e0f962418598e45aa945b18d748e Mon Sep 17 00:00:00 2001 From: infwinston Date: Tue, 20 Dec 2016 11:12:56 +0800 Subject: [PATCH 001/120] Fix an example --- Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk | 1 + 1 file changed, 1 insertion(+) diff --git a/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk b/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk index 11bcf16879a6..f871653d41c8 100644 --- a/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk +++ b/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk @@ -73,6 +73,7 @@ trainNetwork = { syncPeriod = 64 usePipeline = false ] + ] } reader = { From c5c5b9d17bf633adadd900fe121144445d6ef008 Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Tue, 27 Dec 2016 16:24:56 -0800 Subject: [PATCH 002/120] made non-contiguity of data a warning --- bindings/python/cntk/utils/__init__.py | 5 +++-- bindings/python/cntk/utils/tests/utils_test.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/bindings/python/cntk/utils/__init__.py b/bindings/python/cntk/utils/__init__.py index 9162aff00af6..e61bc445ac53 100644 --- a/bindings/python/cntk/utils/__init__.py +++ b/bindings/python/cntk/utils/__init__.py @@ -16,6 +16,7 @@ from .swig_helper import typemap from ..axis import Axis from .progress_print import * +import warnings def sanitize_precision(precision): @@ -572,8 +573,8 @@ def create(var, batch, seq_starts=None, device=None, read_only=False): if isinstance(sample, np.ndarray): if not _is_c_contiguous(sample): - raise ValueError('supplied data is not C contiguous; use ' - 'np.ascontiguousarray (slow) or rearrange your data/computation') + warnings.warn('supplied data is not C contiguous; rearrange your data/computation to avoid this', RuntimeWarning) + sample = np.ascontiguousarray(sample) ndav = _create_NDArrayView_from_NumPy(sample, cpu_dev) elif sparse.issparse(sample): diff --git a/bindings/python/cntk/utils/tests/utils_test.py b/bindings/python/cntk/utils/tests/utils_test.py index 4bd0f5721642..6ffc92454023 100644 --- a/bindings/python/cntk/utils/tests/utils_test.py +++ b/bindings/python/cntk/utils/tests/utils_test.py @@ -112,8 +112,9 @@ def test_sanitize_batch_contiguity(): var = input_variable((2,2), is_sparse=True) batch = [a1.T,a2.T] - with pytest.raises(ValueError): + with pytest.warns(RuntimeWarning): b = sanitize_batch(var, batch) + assert b.shape == (2,1,2,2) batch = [a1,a2] b = sanitize_batch(var, batch) From 1abd7e9fcd270cdbb8c0df1b03c15b3deb52f3a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Gonz=C3=A1lez-Fierro?= Date: Thu, 29 Dec 2016 10:56:13 +0000 Subject: [PATCH 003/120] Fixed bug in ConvNet_MNIST.py When running `ConvNet_MNIST.py` I get the following error: ```bash Traceback (most recent call last): File "ConvNet_MNIST.py", line 131, in convnet_mnist() File "ConvNet_MNIST.py", line 117, in convnet_mnist metric_numer += trainer.test_minibatch(data) * current_minibatch File "/home/hoaphumanoid/anaconda3/envs/cntk-py34/lib/python3.4/site-packages/cntk/trainer.py", line 125, in test_minibatch arguments = sanitize_var_map(self.model.arguments, arguments) File "/home/hoaphumanoid/anaconda3/envs/cntk-py34/lib/python3.4/site-packages/cntk/utils/__init__.py", line 373, in sanitize_var_map len(op_arguments)) ValueError: function expects 1 arguments ``` --- Examples/Image/Classification/ConvNet/Python/ConvNet_MNIST.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Image/Classification/ConvNet/Python/ConvNet_MNIST.py b/Examples/Image/Classification/ConvNet/Python/ConvNet_MNIST.py index d45ce53600b1..9d25826982e4 100644 --- a/Examples/Image/Classification/ConvNet/Python/ConvNet_MNIST.py +++ b/Examples/Image/Classification/ConvNet/Python/ConvNet_MNIST.py @@ -118,7 +118,7 @@ def convnet_mnist(debug_output=False): metric_denom += current_minibatch # Keep track of the number of samples processed so far. - sample_count += trainer.previous_minibatch_sample_count + sample_count += data[label_var].num_samples minibatch_index += 1 print("") From 45de508129eedfca3b04c3407e159137d11d1683 Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Mon, 2 Jan 2017 17:10:44 +0100 Subject: [PATCH 004/120] This commit intentionally left empty. From 9ca092a209b6002793cba6a90a89b9650c233669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miguel=20Gonz=C3=A1lez-Fierro?= Date: Tue, 3 Jan 2017 12:52:09 +0000 Subject: [PATCH 005/120] Bug in script --- Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk | 1 + 1 file changed, 1 insertion(+) diff --git a/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk b/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk index 25dbbc92eb81..8095eab8c2e4 100644 --- a/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk +++ b/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk @@ -74,6 +74,7 @@ trainNetwork = { syncPeriod = 64 usePipeline = false ] + ] } reader = { From 7308637280c6a5b116f75469a58cf994fc894bf6 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Wed, 4 Jan 2017 15:54:03 +0100 Subject: [PATCH 006/120] fix bug in ValueBuffer constructor: setting Size correctly; Add tests update baseline file --- .../EvalWrapper/EvalExtendedWrapper.cpp | 2 +- .../UnitTests/ManagedEvalTests/baseline.txt | 3 +- .../ManagedEvalTests/EvalManagedTests.cs | 58 ++++++++++++++++++- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/Source/Extensibility/EvalWrapper/EvalExtendedWrapper.cpp b/Source/Extensibility/EvalWrapper/EvalExtendedWrapper.cpp index e90a37579026..56b12a4e3404 100644 --- a/Source/Extensibility/EvalWrapper/EvalExtendedWrapper.cpp +++ b/Source/Extensibility/EvalWrapper/EvalExtendedWrapper.cpp @@ -71,7 +71,7 @@ public ref class ValueBuffer Buffer = gcnew cli::array(bufferSize); Indices = gcnew cli::array(bufferSize); ColIndices = gcnew cli::array(colIndicesSize); - Size = colIndicesSize - 1; + Size = colIndicesSize; } // diff --git a/Tests/EndToEndTests/UnitTests/ManagedEvalTests/baseline.txt b/Tests/EndToEndTests/UnitTests/ManagedEvalTests/baseline.txt index f1107067e754..ad89ad8ca13b 100644 --- a/Tests/EndToEndTests/UnitTests/ManagedEvalTests/baseline.txt +++ b/Tests/EndToEndTests/UnitTests/ManagedEvalTests/baseline.txt @@ -16,11 +16,12 @@ Passed EvalManagedVariableSchemaTest Passed EvalManagedConstantNetworkTest Passed EvalManagedScalarTimesTest Passed EvalManagedSparseTimesTest +Passed EvalManagedUsingSparseValueBufferTest Passed EvalManagedScalarTimesDualOutputTest Passed EvalManagedCrossAppDomainExceptionTest Passed EvalManagedImageApiErrorHandling Passed EvalManagedRNNTest -Total tests: 13. Passed: 13. Failed: 0. Skipped: 0. +Total tests: 14. Passed: 14. Failed: 0. Skipped: 0. Test Run Successful. Test execution time: 1.0004 Seconds \ No newline at end of file diff --git a/Tests/UnitTests/ManagedEvalTests/EvalManagedTests.cs b/Tests/UnitTests/ManagedEvalTests/EvalManagedTests.cs index 5c29cb799dff..dbf4469ce8f3 100644 --- a/Tests/UnitTests/ManagedEvalTests/EvalManagedTests.cs +++ b/Tests/UnitTests/ManagedEvalTests/EvalManagedTests.cs @@ -210,6 +210,7 @@ public void EvalManagedValuesBufferTest() Assert.AreEqual(bufferSize, vb.Buffer.Length); Assert.AreEqual(bufferSize, vb.Indices.Length); Assert.AreEqual(colIndicesSize, vb.ColIndices.Length); + Assert.AreEqual(colIndicesSize, vb.Size); } [TestMethod] @@ -310,12 +311,12 @@ public void EvalManagedScalarTimesTest() [TestMethod] public void EvalManagedSparseTimesTest() { - string modelDefinition = @"deviceId = -1 + string modelDefinition = @"deviceId = -1 precision = ""float"" traceLevel = 1 run=NDLNetworkBuilder - NDLNetworkBuilder=[ + NDLNetworkBuilder=[ i1 = SparseInput(3) - o1 = Times(Constant(2, rows=1, cols=3), i1, tag=""output"") + o1 = Times(Constant(2, rows=1, cols=3), i1, tag=""output"") FeatureNodes = (i1) ]"; @@ -359,6 +360,57 @@ public void EvalManagedSparseTimesTest() } } + [TestMethod] + public void EvalManagedUsingSparseValueBufferTest() + { + string modelDefinition = @"deviceId = -1 + precision = ""float"" traceLevel = 1 + run=NDLNetworkBuilder + NDLNetworkBuilder=[ + i1 = SparseInput(3) + o1 = Times(Constant(2, rows=1, cols=3), i1, tag=""output"") + FeatureNodes = (i1) + ]"; + + using (var model = new ModelEvaluationExtendedF()) + { + model.CreateNetwork(modelDefinition); + + VariableSchema outputSchema = model.GetOutputSchema(); + model.StartForwardEvaluation(outputSchema.Select(s => s.Name).ToList()); + + var outputDataLength = 3; + var outputBuffer = new[] + { + new ValueBuffer(outputDataLength) + }; + + var inputData = new float[] { 1, 2, 3, 5, 6 }; + var inputIndices = new [] { 0, 2, 2, 1, 2 }; + var inputColIndices = new [] { 0, 2, 2, 5 }; + + var inputBuffer = new[] + { + new ValueBuffer(inputData.Length, inputColIndices.Length) + }; + + inputData.CopyTo(inputBuffer[0].Buffer, 0); + inputIndices.CopyTo(inputBuffer[0].Indices, 0); + inputColIndices.CopyTo(inputBuffer[0].ColIndices, 0); + + // We can call the evaluate method and get back the results... + model.ForwardPass(inputBuffer, outputBuffer); + + float[][] expected = { new float[] { 6, 0, 28 } }; + + Assert.AreEqual(expected.Length, outputBuffer.Length); + for (int idx = 0; idx < expected.Length; idx++) + { + CollectionAssert.AreEqual(expected[idx], outputBuffer[idx].Buffer); + } + } + } + [TestMethod] public void EvalManagedScalarTimesDualOutputTest() { From 0189a2c51d835f2d2f9736a1cb250f034336724a Mon Sep 17 00:00:00 2001 From: Simon Layton Date: Wed, 27 Jul 2016 13:22:41 -0400 Subject: [PATCH 007/120] Initial divmod implementation --- Source/Math/GPUTensor.cu | 140 ++++++++++++++++++++++++++++++-------- Source/Math/fast_divmod.h | 139 +++++++++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+), 30 deletions(-) create mode 100644 Source/Math/fast_divmod.h diff --git a/Source/Math/GPUTensor.cu b/Source/Math/GPUTensor.cu index 389107167e69..7a7ff5975cb1 100644 --- a/Source/Math/GPUTensor.cu +++ b/Source/Math/GPUTensor.cu @@ -1,5 +1,6 @@ // // Copyright (c) Microsoft. All rights reserved. +// Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved // Licensed under the MIT license. See LICENSE.md file in the project root for full license information. // @@ -15,12 +16,16 @@ #include "CommonMatrix.h" #define TENSOR_OPS_DECL __device__ __host__ #include "TensorOps.h" +#include "fast_divmod.h" #include #include #include "cublas_v2.h" #include #include +// use fast divisor +#define USE_FAST_DIVMOD + #ifndef let #define let const auto #endif @@ -339,8 +344,10 @@ struct TensorOpReduce for (C_size_t k = 1 /*done with k=0 already*/; k < dim; k++) { // bump the pointers - for (C_size_t i = 0; i < N - 1; i++) // N-1 because output is not used here + #pragma unroll + for (C_size_t i = 0; i < N - 1; i++) { // N-1 because output is not used here pointers[i] += reducingStrides(i, (C_size_t) m); + } ElemType val = TensorOpReduce::Compute(pointers, op, reductionOp, reducingOpDims, reducingStrides); UpdateAggregate(aggregate, val, reductionOp); } @@ -372,18 +379,35 @@ struct TensorOpParallelReduce { // this version for m >= 0 static __device__ ElemType Compute(CUDA_LONG id, FixedArray pointers, ElementWiseOperator op, - const FixedArray& reducingOpDims, const FixedMatrix& reducingStrides) + const FixedArray& reducingOpDims, const FixedMatrix& reducingStrides, + FixedArray reducingOpDimDivmod) { // map id (location on grid) to index[k] C_size_t stride = 1; // compute the stride. This seems expensive, but since we we only currently support M <= 2, this is just compile-time selection between 1 and reducingOpDims[0]. - for (int i = 0; i < m; i++) + #pragma unroll + for (int i = 0; i < m; i++) { stride *= reducingOpDims[(C_size_t) i]; - C_size_t index = id / stride; // this dimension. For m=0, the stride is 1 and hence the division will be removed at compile time. - id = id % stride; // remaining dimensions inside this. For m=0 this value is ignored and hence not even computed. + } + + C_size_t index; +#ifndef USE_FAST_DIVMOD + index = id / stride; // this dimension. For m=0, the stride is 1 and hence the division will be removed at compile time. + // id = id % stride; // remaining dimensions inside this. For m=0 this value is ignored and hence not even computed. + id = id - stride*index; // remaining dimensions inside this. For m=0 this value is ignored and hence not even computed. +#else + if (m == 0) { + index = id; + id = 0; + } else { + reducingOpDimDivmod[m].divmod(id, index, id); + } +#endif // apply this index to the pointers - for (C_size_t i = 0; i < N - 1; i++) + #pragma unroll + for (C_size_t i = 0; i < N - 1; i++) { pointers[i] += index * reducingStrides(i, (C_size_t) m); // now this dimension is taken care of - return TensorOpParallelReduce::Compute(id, pointers, op, reducingOpDims, reducingStrides); + } + return TensorOpParallelReduce::Compute(id, pointers, op, reducingOpDims, reducingStrides, reducingOpDimDivmod); } }; @@ -395,7 +419,8 @@ struct TensorOpParallelReduce // this version for m = -1 // the pointers are pointing to the right location(s) to take the operation over static __device__ ElemType Compute(CUDA_LONG /*id*/, FixedArray pointers, ElementWiseOperator op, - const FixedArray& /*reducingOpDims*/, const FixedMatrix& /*reducingStrides*/) + const FixedArray& /*reducingOpDims*/, const FixedMatrix& /*reducingStrides*/, + FixedArray reducingOpDimDivmod) { return TensorOps::Compute(pointers, op); // finally computing something! } @@ -413,17 +438,27 @@ struct TensorOpElement static __device__ void Compute(CUDA_LONG id, ElemType beta, FixedArray& pointers, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, const FixedArray& regularOpStrides, const FixedMatrix& regularStrides, const FixedArray& reducingOpDims, const FixedMatrix& reducingStrides, - CUDA_LONG reductionBegin, CUDA_LONG reductionChunkSize) + CUDA_LONG reductionBegin, CUDA_LONG reductionChunkSize, + FixedArray regularOpStrideDivmod, FixedArray reducingOpDimDivmod) { // map id (location on grid) to index[k] +#ifndef USE_FAST_DIVMOD C_size_t stride = regularOpStrides[(C_size_t) k]; C_size_t index = id / stride; // this dimension - id = id % stride; // remaining dimensions inside this + // id = id % stride; // remaining dimensions inside this + id = id - stride*index; // remaining dimensions inside this +#else + C_size_t index; + regularOpStrideDivmod[k].divmod(id, index, id); +#endif // apply this index to the pointers - for (C_size_t i = 0; i < N; i++) + #pragma unroll + for (C_size_t i = 0; i < N; i++) { pointers[i] += index * regularStrides(i, (C_size_t) k); // now this dimension is taken care of + } // process the previous index - TensorOpElement::Compute(id, beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, reducingOpDims, reducingStrides, reductionBegin, reductionChunkSize); + TensorOpElement::Compute(id, beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, reducingOpDims, reducingStrides, reductionBegin, reductionChunkSize, + regularOpStrideDivmod, reducingOpDimDivmod); } }; @@ -435,15 +470,19 @@ struct TensorOpElement static __device__ void Compute(CUDA_LONG id, ElemType beta, FixedArray& pointers, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, const FixedArray& regularOpStrides, const FixedMatrix& regularStrides, const FixedArray& reducingOpDims, const FixedMatrix& reducingStrides, - CUDA_LONG reductionBegin, CUDA_LONG reductionChunkSize) + CUDA_LONG reductionBegin, CUDA_LONG reductionChunkSize, + FixedArray regularOpStrideDivmod, FixedArray reducingOpDimDivmod) { // map id (location on grid) to index[k] C_size_t index = id; // this dimension // apply this index to the pointers - for (C_size_t i = 0; i < N; i++) + #pragma unroll + for (C_size_t i = 0; i < N; i++) { pointers[i] += index * regularStrides(i, 0); // now this dimension is taken care of + } // process the previous index - TensorOpElement::Compute(/*id*/ 0, beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, reducingOpDims, reducingStrides, reductionBegin, reductionChunkSize); + TensorOpElement::Compute(/*id*/ 0, beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, reducingOpDims, reducingStrides, reductionBegin, reductionChunkSize, + regularOpStrideDivmod, reducingOpDimDivmod); } }; @@ -455,7 +494,8 @@ struct TensorOpElement // now the output pointers point to the right element (input pointers may still iterate for reduction) static __device__ void Compute(CUDA_LONG /*id*/, ElemType beta, FixedArray& pointers, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, const FixedArray& /*regularOpStrides*/, const FixedMatrix& /*regularStrides*/, - const FixedArray& reducingOpDims, const FixedMatrix& reducingStrides, CUDA_LONG /*reductionBegin*/, CUDA_LONG /*reductionChunkSize*/) + const FixedArray& reducingOpDims, const FixedMatrix& reducingStrides, CUDA_LONG /*reductionBegin*/, CUDA_LONG /*reductionChunkSize*/, + FixedArray regularOpStrideDivmod, FixedArray reducingOpDimDivmod) { // compute the operation for this output coordinate // This may still involve a reduction over inverse-broadcasting dimensions. @@ -484,7 +524,8 @@ struct TensorOpElement // now the output pointers point to the right element (input pointers may still iterate for reduction) static __device__ void Compute(CUDA_LONG /*id*/, ElemType beta, FixedArray& pointers, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, const FixedArray& /*regularOpStrides*/, const FixedMatrix& /*regularStrides*/, - const FixedArray& reducingOpDims, const FixedMatrix& reducingStrides, CUDA_LONG reductionBegin, CUDA_LONG reductionChunkSize) + const FixedArray& reducingOpDims, const FixedMatrix& reducingStrides, CUDA_LONG reductionBegin, CUDA_LONG reductionChunkSize, + FixedArray regularOpStrideDivmod, FixedArray reducingOpDimDivmod) { CUDA_LONG reductionBlock = blockIdx.z; // reduction-block index --larger reductions are split into blocks CUDA_LONG tid = threadIdx.x; // thread index @@ -505,7 +546,7 @@ struct TensorOpElement for (CUDA_LONG redId = reductionBegin + tid; redId < reductionEnd; redId += tids) { - auto val = TensorOpParallelReduce::Compute(redId, pointers, op, reducingOpDims, reducingStrides); + auto val = TensorOpParallelReduce::Compute(redId, pointers, op, reducingOpDims, reducingStrides, reducingOpDimDivmod); UpdateAggregate(aggregate, val, reductionOp); } @@ -562,11 +603,13 @@ struct TensorOpElement template __global__ void _launchTensorOp(ElemType beta, FixedArray pointers, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, FixedArray regularOpStrides, FixedMatrix regularStrides, CUDA_LONG numElements, - FixedArray reducingOpDims, FixedMatrix reducingStrides) + FixedArray reducingOpDims, FixedMatrix reducingStrides, + FixedArray regularOpStrideDivmod, FixedArray reducingOpDimDivmod) { CUDA_LONG id = GridDim::GetLinearThreadId(); if (id < numElements) // note: there are no __syncthread() calls inside - TensorOpElement::Compute(id, beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, reducingOpDims, reducingStrides, 0, 0); + TensorOpElement::Compute(id, beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, reducingOpDims, reducingStrides, 0, 0, + regularOpStrideDivmod, reducingOpDimDivmod); } template @@ -577,21 +620,33 @@ static void LaunchTensorOp(ElemType beta, array pointerVector, Ele FixedArray pointers(pointerVector); SmallVector regularOpStrideVector; // kernel needs the strides for converting thread index back to multi-dimensional tensor index C_size_t numElements = 1; + // input divisors + SmallVector regularOpStrideDivmodVector; for (C_size_t k = 0; k < regularOpDims.size(); k++) { regularOpStrideVector.push_back(numElements); + // create fast division objects + regularOpStrideDivmodVector.push_back(fast_divmod(numElements)); numElements *= (C_size_t) regularOpDims[k]; } + + SmallVector reducingOpDimDivmodVector; + FixedArray regularOpStrides(regularOpStrideVector); FixedMatrix regularStrides(regularStrideVectors); FixedArray reducingOpDims; // empty reduction dimensions FixedMatrix reducingStrides; + // reduced divisors + FixedArray regularOpStrideDivmod(regularOpStrideDivmodVector); + FixedArray reducingOpDimDivmod; // launch the kernel CUDA_LONG NN = (CUDA_LONG) numElements; // linear space identifying each individual input element SyncGuard syncGuard; GridDim grid(NN); - _launchTensorOp <<>>(beta, pointers, alpha, op, (ElementWiseOperator)(-1) /* dummy reductionOp */, regularOpStrides, regularStrides, grid.m_N, reducingOpDims, reducingStrides); + _launchTensorOp <<>>(beta, pointers, alpha, op, (ElementWiseOperator)(-1) /* dummy reductionOp */, regularOpStrides, regularStrides, + grid.m_N, reducingOpDims, reducingStrides, + regularOpStrideDivmod, reducingOpDimDivmod); } // ----------------------------------------------------------------------- @@ -602,15 +657,18 @@ template __global__ void _launchTensorOpWithReduction(ElemType beta, FixedArray pointers, ElemType alpha, ElementWiseOperator op, ElementWiseOperator reductionOp, FixedArray regularOpStrides, FixedMatrix regularStrides, CUDA_LONG numElements, FixedArray reducingOpDims, FixedMatrix reducingStrides, - CUDA_LONG reductionBegin, CUDA_LONG reductionChunkSize) + CUDA_LONG reductionBegin, CUDA_LONG reductionChunkSize, + FixedArray regularOpStrideDivmod, FixedArray reducingOpDimDivmod) { CUDA_LONG id = gridDim.x * blockIdx.y + blockIdx.x; // input dimensions are Y dimension of blocks in this case, so we can use thread dim for shared-memory/parallelization #ifndef ALLOW_ATOMIC_REDUCTION CUDA_LONG reductionBlock = blockIdx.z; // reduction-block index --larger reductions are split into blocks pointers[pointers.size() - 1] += numElements * reductionBlock; // the output tensor is dense (no gaps); and there is one copy for each reduction block (those get further reduced into one later) #endif - if (id < numElements) // note: we have __syncthread() calls but only entire blocks in sync, so this is OK - TensorOpElement::Compute(id, beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, reducingOpDims, reducingStrides, reductionBegin, reductionChunkSize); + if (id < numElements) { // note: we have __syncthread() calls but only entire blocks in sync, so this is OK + TensorOpElement::Compute(id, beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, reducingOpDims, reducingStrides, reductionBegin, reductionChunkSize, + regularOpStrideDivmod, reducingOpDimDivmod); + } } // helper function to provide a reduction buffer @@ -654,15 +712,29 @@ static void LaunchTensorOpWithReduction(ElemType beta, array point FixedArray pointers(pointerVector); SmallVector regularOpStrideVector; // kernel needs the strides for converting thread index back to multi-dimensional tensor index C_size_t numElements = 1; + // input divisors + SmallVector regularOpStrideDivmodVector; for (C_size_t k = 0; k < regularOpDims.size(); k++) { regularOpStrideVector.push_back(numElements); // stride for dense representation of our output elements (if they were flattened) + regularOpStrideDivmodVector.push_back(fast_divmod((unsigned int)numElements)); numElements *= (C_size_t) regularOpDims[k]; } + // output divisors + SmallVector reducingOpDimDivmodVector; + C_size_t stride = 1; + for (C_size_t k = 0; k < reducingOpDimVector.size(); ++k) { + reducingOpDimDivmodVector.push_back(fast_divmod(stride)); + stride *= reducingOpDimVector[k]; + } + FixedArray regularOpStrides(regularOpStrideVector); FixedMatrix regularStrides(regularStrideVectors); FixedArray reducingOpDims(reducingOpDimVector); FixedMatrix reducingStrides(reducingStrideVectors); + // reduced divisors + FixedArray regularOpStrideDivmod(regularOpStrideDivmodVector); + FixedArray reducingOpDimDivmod(reducingOpDimDivmodVector); // launch the kernel CUDA_LONG NN = (CUDA_LONG) numElements; // linear space identifying each individual output element @@ -697,7 +769,8 @@ static void LaunchTensorOpWithReduction(ElemType beta, array point _launchTensorOp<<>>( beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, grid.m_N, - reducingOpDims, reducingStrides); + reducingOpDims, reducingStrides, + regularOpStrideDivmod, reducingOpDimDivmod); } // === optimization: simple case would not use all multiprocs else @@ -750,7 +823,8 @@ static void LaunchTensorOpWithReduction(ElemType beta, array point _launchTensorOpWithReduction<<>>( beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, NN, - reducingOpDims, reducingStrides, /*reductionBegin*/ 0, reductionChunkSize); + reducingOpDims, reducingStrides, /*reductionBegin*/ 0, reductionChunkSize, + regularOpStrideDivmod, reducingOpDimDivmod); } // --- case (b) // Reduction across blocks. This is the difficult one. @@ -787,7 +861,8 @@ static void LaunchTensorOpWithReduction(ElemType beta, array point _launchTensorOpWithReduction << > >( beta1, pointers1, alpha1, op, reductionOp, regularOpStrides, regularStrides1, NN, - reducingOpDims, reducingStrides, /*reductionBegin*/0, reductionChunkSize); + reducingOpDims, reducingStrides, /*reductionBegin*/0, reductionChunkSize, + regularOpStrideDivmod, reducingOpDimDivmod); #if 1 // now reduce and redistribute @@ -805,6 +880,7 @@ static void LaunchTensorOpWithReduction(ElemType beta, array point beta, pointerVector2, alpha, ElementWiseOperator::opCopy, reductionOp, regularOpDims, regularStrideVectors2, reducingOpDimVector2, reducingStrideVectors2); + // (note: ^^this will have a nested syncGuard, which is fine) #else @@ -832,16 +908,20 @@ static void LaunchTensorOpWithReduction(ElemType beta, array point else if (beta == 1) { // no need to pre-scale; just add (common for gradients) - _launchTensorOpWithReduction<<>>(beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, NN, reducingOpDims, reducingStrides, 0, reductionChunkSize); + _launchTensorOpWithReduction<<>>(beta, pointers, alpha, op, reductionOp, regularOpStrides, + regularStrides, NN, reducingOpDims, reducingStrides, 0, reductionChunkSize, + regularOpStrideDivmod, reducingOpDimDivmod); return; } else { // We need more than one chunk, we will use atomicAdd(). // First reset/pre-multiply input; then do the remaining chunks using atomicAdd(). - _launchTensorOpWithReduction<<>>(beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, NN, reducingOpDims, reducingStrides, 0, reductionChunkSize); + _launchTensorOpWithReduction<<>>(beta, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, NN, reducingOpDims, reducingStrides, 0, reductionChunkSize, + regularOpStrideDivmod, reducingOpDimDivmod); // We will leave it like this for a while, but eventually need to revisit using temporary memory. - _launchTensorOpWithReduction<<>>(/*beta=*/1, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, NN, reducingOpDims, reducingStrides, reductionChunkSize, reductionChunkSize); + _launchTensorOpWithReduction<<>>(/*beta=*/1, pointers, alpha, op, reductionOp, regularOpStrides, regularStrides, NN, reducingOpDims, reducingStrides, reductionChunkSize, reductionChunkSize, + regularOpStrideDivmod, reducingOpDimDivmod); } #endif } diff --git a/Source/Math/fast_divmod.h b/Source/Math/fast_divmod.h new file mode 100644 index 000000000000..58590e90fb07 --- /dev/null +++ b/Source/Math/fast_divmod.h @@ -0,0 +1,139 @@ +// +// Copyright (c) 2016, NVIDIA CORPORATION. All rights reserved +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +#pragma once + +#include +#include +#include + +namespace Microsoft { namespace MSR { namespace CNTK { + +namespace detail { + +__host__ __device__ __forceinline__ +int mulhi(const int M, const int n) { +#ifdef __CUDA_ARCH__ + return __mulhi(M, n); +#else + return (((unsigned long long)((long long)M * (long long)n)) >> 32); +#endif +} + +} + +// Based on code from Chapter 10 of "Hacker's Delight, 2nd ed." +class fast_divmod { + public: + fast_divmod(int d = 1) : d_(d), a_(0) { + find_magic_numbers(); + } + __host__ __device__ + fast_divmod(const fast_divmod& other) : d_(other.d_), a_(other.a_), s_(other.s_), M_(other.M_) {}; + __host__ __device__ + int div(int n) { + // get high 32 bits of M * n + int q = detail::mulhi(M_, n); + + // deal with add / subs if needed + q += a_ * n; + + // shift if necessary + if (s_ >= 0) { + q >>= s_; + q += ((unsigned int)q >> 31); + } + + return q; + } + __host__ __device__ + void divmod(int n, int& q, int& r) { + // handle special cases + if (d_ == 1) { + q = n; + r = 0; + } else if (d_ == -1) { + q = -n; + r = 0; + } else { + // general case + q = div(n); + r = n - q*d_; + } + } + // Needed for overflow checks in FixedArray / FixedMatrix + bool operator!=(const fast_divmod& other) { + return (d_ != other.d_) || (M_ != other.M_) || (s_ != other.s_) || (a_ != other.a_); + } + + public: + int d_, M_, s_, a_; + + private: + // Based on code from Hacker's delight 2.ed + // Chapter 10, figure 10-1 + void find_magic_numbers() { + // special case for d = 1, -1 + if (d_ == 1) { + M_ = 0; + s_ = 0; + a_ = 1; + return; + } else if (d_ == -1) { + M_ = 0; + s_ = -1; + a_ = -1; + return; + } + // general case + const unsigned two31 = 0x80000000; + unsigned abs_d = (d_ == 0) ? 1 : abs(d_); + unsigned t = two31 + ((unsigned)d_ >> 31); // t = 2^31 + (d < 0) ? 1 : 0 + unsigned abs_nc = t - 1 - (t % abs_d); // |n_c| = t - 1 - rem(t, |d|) + int p = 31; + unsigned q1 = two31 / abs_nc; // Init q_1 = 2^31 / |n_c| + unsigned r1 = two31 - q1 * abs_nc; // Init r_1 = rem(q_1, |n_c|) + unsigned q2 = two31 / abs_d; // Init q_2 = 2^31 / |d| + unsigned r2 = two31 - q2 * abs_d; // Init r_2 = rem(q_2, |d|) + + unsigned delta; + // iterate p until + // 2^p < n_c * (d - rem(2^p, d)) is satisfied + do { + ++p; + q1 *= 2; + r1 *= 2; + + if (r1 >= abs_nc) { + q1 += 1; + r1 -= abs_nc; + } + q2 *= 2; + r2 *= 2; + + if (r2 >= abs_d) { + q2 += 1; + r2 -= abs_d; + } + delta = abs_d - r2; + } while (q1 < delta || + (q1 == delta && r1 == 0)); + + // store magic numbers + M_ = q2 + 1; + if (d_ < 0) M_ = -M_; + s_ = p - 32; + + // generate sentinel for correct adds / subs + // "generate the add if d > 0 and M < 0" + if ((d_ > 0) && (M_ < 0)) a_ = 1; + // "generate the sub if d < 0 and M > 0" + else if ((d_ < 0) && (M_ > 0)) a_ = -1; + // Otherwise no add / sub needed + else a_ = 0; + } +}; + +}}} // End namespaces From 8a07efdfec275a313a19d7b4ce7a4770c8aca697 Mon Sep 17 00:00:00 2001 From: "REDMOND\\sayanpa" Date: Wed, 4 Jan 2017 13:58:36 -0800 Subject: [PATCH 008/120] bug fix in testing model data source --- .../CNTK_103B_MNIST_FeedForwardNetwork.ipynb | 160 ++++-------------- 1 file changed, 31 insertions(+), 129 deletions(-) diff --git a/Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb b/Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb index fd0a4b6b8d73..c2e4965d76ec 100644 --- a/Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb +++ b/Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb @@ -36,7 +36,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "collapsed": false, "nbpresent": { @@ -73,7 +73,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "collapsed": true }, @@ -98,19 +98,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Training data from file data/MNIST/Train-28x28_cntk_text.txt successfully read.\n" - ] - } - ], + "outputs": [], "source": [ "# Ensure the training data is generated and available for this tutorial\n", "train_file = \"data/MNIST/Train-28x28_cntk_text.txt\"\n", @@ -156,7 +148,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "collapsed": true }, @@ -180,7 +172,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "collapsed": false }, @@ -206,7 +198,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "collapsed": true }, @@ -258,7 +250,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": null, "metadata": { "collapsed": false }, @@ -294,7 +286,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": { "collapsed": false }, @@ -314,7 +306,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": null, "metadata": { "collapsed": false }, @@ -345,7 +337,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": null, "metadata": { "collapsed": false }, @@ -367,7 +359,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": null, "metadata": { "collapsed": false }, @@ -411,7 +403,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": null, "metadata": { "collapsed": false }, @@ -426,37 +418,11 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Minibatch: 0, Loss: 2.3476, Error: 89.06%\n", - "Minibatch: 500, Loss: 0.2800, Error: 9.38%\n", - "Minibatch: 1000, Loss: 0.0348, Error: 0.00%\n", - "Minibatch: 1500, Loss: 0.1608, Error: 7.81%\n", - "Minibatch: 2000, Loss: 0.0722, Error: 3.12%\n", - "Minibatch: 2500, Loss: 0.0399, Error: 1.56%\n", - "Minibatch: 3000, Loss: 0.0520, Error: 3.12%\n", - "Minibatch: 3500, Loss: 0.0186, Error: 0.00%\n", - "Minibatch: 4000, Loss: 0.0181, Error: 0.00%\n", - "Minibatch: 4500, Loss: 0.0269, Error: 0.00%\n", - "Minibatch: 5000, Loss: 0.0506, Error: 1.56%\n", - "Minibatch: 5500, Loss: 0.0280, Error: 1.56%\n", - "Minibatch: 6000, Loss: 0.0019, Error: 0.00%\n", - "Minibatch: 6500, Loss: 0.0858, Error: 3.12%\n", - "Minibatch: 7000, Loss: 0.0022, Error: 0.00%\n", - "Minibatch: 7500, Loss: 0.0020, Error: 0.00%\n", - "Minibatch: 8000, Loss: 0.0006, Error: 0.00%\n", - "Minibatch: 8500, Loss: 0.0006, Error: 0.00%\n", - "Minibatch: 9000, Loss: 0.0018, Error: 0.00%\n" - ] - } - ], + "outputs": [], "source": [ "# Run the trainer on and perform model training\n", "training_progress_output_freq = 500\n", @@ -489,32 +455,11 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAADeCAYAAADmUqAlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3XecXFX9//HXO10CJNQENHSCFAkk4EYEUcEfAgKCtEWk\ngxAQjH7FL19UwAKIAoIIFqTDCmILRZEuRdouKCWEGqoEEBJKEgjZz++PcwduJrNttsyd3ffz8biP\nzJzbPmdmNvOZc885VxGBmZmZWVEMqnUAZmZmZnlOTszMzKxQnJyYmZlZoTg5MTMzs0JxcmJmZmaF\n4uTEzMzMCsXJiZmZmRWKkxMzMzMrFCcnZmZmVihOTqxfkNQq6XtV7jtT0nm55/tmx5vYcxFWT9Kq\nWTzfqHUs1jZJd0m6tsp9fydpek/H1MlzVx23WW9xcmKFkUsKWiVt1sY2z2Xrp5WtimypRmuFfXv0\nvg6SGiUd1ZPHtLZJOj/3WWpvOa/jo3Vadz6DQfoc1oLvYWKFM6TWAZhVMA/YC7gzXyhpS+DDwPwK\n+3wIeK/K861D738x7AWsD5zRy+ex5JfA9bnnqwPfB34N3JYrf7IHz7kF1X/R7w2oB2Mxq2tOTqyI\nrgV2k3RkROSThr2A+4Dly3eIiHerPVlELKh233olaYmImFvrOHpLRNwN3F16LmkS8APgnxFxWWeO\nIWlERFRKhNs6Z7XJMRGxsNp9zfojX9axogmgCVgO+FypUNJQYFfgMir8wizvcyLp+KxsTUkXSHpd\n0mxJ50kaUbbvzDaa90dK+pWkVyXNkXShpNFl++4o6WpJL0iaL+kJSd+RNCi3zc3A9kCp70irpKdy\n64dn8c6QNE/Si5L+IGn1CvU8ODvHfEn3SNqkoxc0d7nsU5LOljQLeC5bd4Gkpyvsc7yk1rKyVkln\nStpJ0oNZDA9J2qaD868oaYGk71ZYNz477pTs+RBJx0l6LHstXpV0m6StOqpnd0h6SdIVkraX1Cxp\nPrBPtu5gSTdJmpXF9KCkAyocY5G+G5K2yeq2Y/Z6viBprqTrJK1atu8ifU4krVN6XbLlyezcd0qa\nUOHce0manm3zQFaPqvuxSBqTfTZezo55v6TGCtvtI6lF0pvZ39e/JB2WWz9M0g8lPZ4d5xVJt0r6\nVDVx2cDhlhMropnAXUAjcF1Wth2wNPA7oDN9N0rN61cATwH/C0wEDgJmAcdU2DZPwFnA68BxpEs/\nU4BVgM/kttsPeBM4FXgL+Czp8sFSwLezbX4IjCJdkvp6duy3ALIk5prsmE3Az7J9PwdsAOQThy8D\nS5IuWUR2/D9IWqOTv7zPBl4GTgCWyNW9Uv3bKt8C2CU71pvAkcCVklaJiNcrnTQiXpZ0K7A7qfUi\nb0/S5bgrsucnkN6rXwP3kt7zTUjv3Y0dV7FqAWwIXEiq2y+Bh7N1U7JY/kS6/PdF4FxJERHnlx2j\nkuOAd4CTSUn30cAFLPo5auv1PhAYAfwCGEx6z6+UND4iAkDSLsAlpFbFb5NaFi8GXmwnpjZJGgnc\nTvq8ngk8D+wBXCppyYj4TbbdDlk9/gr8ivRjd33gE8A52eFOIv29ngPcT/o7+DiwEfCPrsZmA0hE\nePFSiAXYF1hI+iKaAswGhmfrLgduyB4/DUwr27cV+F7u+XFZ2a/LtvsD8HJZ2dPAeWVxtJIuCwzO\nlf9PFt8XcmXDK9TjHNIX99Bc2VXAUxW23T8715HtvC6rZtu8DCydK98hi2e7TryurcAtgMrWnd9G\nXMcBCyu8xvOA1XJlH8vKp3QQw8FZrOuVlT8EXJ97fn/5e9tDn61JWZz7tLH+P1l8m1dYV+k9vgl4\nsKzsn8C1uefbZOdsKfscfSs71xq5sibgkdzzUj+oF4CRufLdsn0/myubATyej5OU3Lbmj9nOa1Me\n97ezc3wxVzaElPz8FxiR+5y/1MGxpwNX9PT76aX/L76sY0V1BenX/RckLQl8Abi0i8cI0i+6vNuA\n5bJjduTXsWiLxDlkycD7J4h4p/RY0pKSliP96lwC+GgnzrEL8AqplaYjv4uIN3LPbyO1wqzRiX0D\n+E1EdHdkxvURMfP9g0Y8CLzRiRj+SHrt9igVSFofWI/UGlYyG1hf0lrdjLMa0yPi9vLCsvd4lKTl\nSb/615U0rBPHPbfsc1TqkNuZ9+3SiHi7bN/33/Ps0t/awPn5OCPielLCUo1tgWci4s+5470H/BwY\nDZRG0s0GRkn6bDvHmg1sWOkSpVl7nJxYIUXEq8ANpE6wu5A+q1dWcahny56XLj0s01EIwBNlMb1N\n+oW9WqlM0nqS/iRpNulL+hVSkzqkJuyOrAnMiEU7/rblubJ4ZmcPO6pLycxObtfpGDKvdxRDRPyX\ndFlm91zxnsAC0uWSku+RvgAfk/RvSadI+lj3Qu60xfreQBolJulmSW+T6vpyFqdIl506Uv6avZ7t\n25n3rdK+5PYt9V2pNOroiQplnbEq8FiF8umkuEvn/DnwDHC9pGck/UbS1mX7HAuMAZ7M+sKcJGm9\nKuOyAcTJiRXZZaRWikOBv0bEm1Uco62+GN0etilpFOkX9MeA75Bad7bmg74mPf331d26zKtQ1lZL\nyuBeiOF3wHhJG2bPdwNujIjX3g8m4jZSwrY/8CCpz0VLpQ6ovWCx10fSR4G/AyNJfSe2I73HpZau\nzrzH3XnNeu3z210R8SLps78zqd/U1sDfJZ2T2+Ym0vt5ICm5+SrwgKQv933EVk+cnFiRlTogNpAS\nlb4kUnP5BwWpo+BKfNAC8WnSL9h9I+KsiLg2+894NotrKwl4ElhHUlvJQG97ndRSUW61XjjXn0kt\nJXtkI07Gk/paLCIiZkfEhRHxZWAc8G/g+F6IpzN2IvW32C4izo2Iv2XvcVGGnz+T/VvpMli1l8ae\nIb035dYlfY5L5yQiFkTEtIiYQrrUdAFwiKSVc9u8FhHnR0QjqUP5DFKfJrM2OTmxwsouoxxK+mK6\nqgYhHCIpP6JtCqlFoTRcdCEpickPGx6WbVfubSpf5vkDsAJwRE8EXIUnSf0GNigVSFqJNCKlR0XE\nHNLoq91Jl3TeAf6S30bSsmX7zCVdnhie22bpbKhtZy6pdFep5SL/Hi9HmjStM3p19tWIeJrUt2Q/\n5YbIKw3vXrvNHdt3LWnY+0654w0hfUZnA3dkZeXvVZA6OEP2flXY5i3S6LnhmLXDQ4mtaBZpro6I\ni9vasA8MA26UdAWpc+thwG0RcXW2/k5Sy8NFks7Myvam8hdSM7C7pFNJw1Lfyo5zEWk+jdMkNZA6\nPC4JbAX8IiJ6Kilr6zLA74AfA3/O6jCSlBDOII2a6mmXk4a9TgGuK+vgC/CIpFtIr9drwKak+W3O\nzG2zM2mU0X6k1683/Q04EfirpHNJrUyHkEbRLDYZYAV9cfnlWNLreruki4AVSZ/Vh6nuB+gvSEPu\nL5N0Fqnfy56kz8OhuY63l0gaThoF9gKp5eQI4O4saYLU1+SvpBFLr5OGGX8BOKWKuGwAcXJiRdOZ\nX5qV5oTo7n1NKh3vCNLcIicAQ0mjhd6fYyUiXpO0PWmOkx+Q/vO9mDTM9Lqy450NTCB9oX6d1DR+\ndUS0StqW9AVT6vz7X1KS8mAn6tfZelfcJqvDF4HTSEnK06R5RsazeHLS3RgAppH6doxk0VE6JWcA\nO5KGwg4nvU7/B/y0wjm7qr19KtYhIh6StBvp/T2V9CV8OqnV5+xOnKOtc7b1OnZ23/fXRcSVkr4C\nfJf0Hs4gJclTgJUrHqGdc0fE25K2IM3Lsj9p3p3pwJcjIv+eXUDqSzKFlLT9h5QsnpDb5nTSBITb\nkN7Pp0lDqX/WybhsgFL3RxaamVnRZLPDPhYRO3W4sVnB1LzPiaRjlKbhfkNpeug/SarUGSu/z5Za\n/O6iCyWt2Fdxm5kVgdKU/4PKyj5Pmsjt5tpEZdY9RbisswVpvPx9pHhOIg1HWzciKg19LAlS0/P7\nw0sj4uXeDNTMrIDWJPUZaiJdWlmfNGT3GeC3tQzMrFo1T04iYrv8c0n7kSY5mkSaabM9r1ToUGdm\nNpC8QhpufQipk+4bpBl5j6lybiCzmqt5clLBaFKryGsdbCfSZD4jSMPXjo+IO3s7ODOzIskmsduj\nww3N6kihOsRKEmk+i6UiYst2thsPbEm6FDScdFOxrwAfj4gH+iJWMzMz6x1FS07OIQ05+2RE/KeL\n+95CulnVvm2sXy479kxgfvciNTMzG1BGkGaOvi67V1avKsxlnWyyn+2ALbqamGTuAT7Zzvpt6Ppd\nbc3MzOwDX6YPbidSiOQkS0x2AraMiPK7yHbWRqSe6m2ZCXDJJZew7rrrVnmK4pg6dSqnn356rcPo\nMa5PcfWnuoDrU2T9qS7Qv+ozffp09t57b+iZu5t3qObJiaSzgUbSrJBvSxqTrZoTEfOzbU4EPly6\nZCPpKNJMgw+TmpoOBj5DmlWyLfMB1l13XSZO7I1ZufvWqFGj+kU9Slyf4upPdQHXp8j6U12g/9Un\n0yfdImqenJDu4xGk+zPk7c8H981YiXR30pJhpKmkVwbmkobRbRUR/+jVSM3MzKzX1Tw5iYgOZ6mN\niP3Lnv8E+EmvBWVmZmY1U/Pp683MzMzynJzUqcbGxlqH0KNcn+LqT3UB16fI+lNdoP/Vpy8Vap6T\n3iRpItDc3NzcHzsomZmZ9ZqWlhYmTZoEMCkiWnr7fG45MTMzs0IZcMnJm74NlpmZWaENuOTk4Ydr\nHYGZmZm1Z8AlJw8+WOsIzMzMrD0DLjl56KFaR2BmZmbtGZDJyQAZoGRmZlaXBlxy8oMfODkxMzMr\nsgGXnGy2GQwacLU2MzOrH/6aNjMzs0JxcmJmZmaF4uTEzMzMCsXJiZmZmRWKkxMzMzMrFCcnZmZm\nVig1T04kHSPpHklvSJol6U+Sxndiv09LapY0X9Jjkvbt7Dlvvx2+8Y3uxW1mZma9o+bJCbAF8HOg\nAdgaGAr8XdKH2tpB0mrA1cCNwATgDOBcSZ/rzAmfew5OPx1efbV7gZuZmVnPG1LrACJiu/xzSfsB\nLwOTgNvb2O0w4KmIODp7PkPS5sBU4PqOztnQkP695x7Ybrv2tzUzM7O+VYSWk3KjgQBea2ebycAN\nZWXXAZ/ozAlWXx2WXx7uuqu6AM3MzKz3FCo5kSTgZ8DtEfFIO5uOBWaVlc0ClpY0vOPzpNaTu++u\nPlYzMzPrHYVKToCzgfWAPXv7RA0N6bJOa2tvn8nMzMy6ouZ9TkoknQVsB2wREf/pYPOXgDFlZWOA\nNyLinfZ2nDp1KqNGjeKVV2D2bNh6azj44EYaGxurD97MzKyfaGpqoqmpaZGyOXPm9GkMiog+PWHF\nIFJishOwZUQ81YntTwa2jYgJubLLgNHlHWxz6ycCzc3NzUycOJHZs2GZZeDCC2GffXqoImZmZv1Q\nS0sLkyZNApgUES29fb6aX9aRdDbwZWAv4G1JY7JlRG6bEyVdmNvtl8Aakn4saR1JU4BdgdM6e97R\no+Hww2Hs2B6qiJmZmfWIIlzWOZQ0OueWsvL9gYuyxysB40orImKmpO2B04EjgeeBAyOifARPu846\nq8qIzczMrNfUPDmJiA5bbyJi/wpl/yDNhWJmZmb9SM0v65iZmZnlOTkxMzOzQnFyYmZmZoXi5MTM\nzMwKxcmJmZmZFcqAT04i4I474PHHax2JmZmZgZMTAHbbDc47r9ZRmJmZGTg58R2KzczMCmbAJycA\nkyfDvffCwoW1jsTMzMycnJBaTt56C6ZPr3UkZmZm5uQE2GQTGDTIl3bMzMyKwMkJsOSSsP76cNdd\ntY7EzMzMnJxkJk92y4mZmVkRODnJNDTA/PmwYEGtIzEzMxvYnJxkDjgAHnsMhg6tdSRmZmYDm5OT\njFTrCMzMzAycnJiZmVnBFCI5kbSFpGmSXpDUKmnHDrbfMtsuvyyUtGJfxWxmZma9oxDJCTASeACY\nAkQn9wlgbWBstqwUES/3TnhmZmbWV4bUOgCAiPgb8DcAqUu9P16JiDd6JyozMzOrhaK0nFRDwAOS\nXpT0d0mb1TogMzMz6756TU7+A3wV+BKwC/AccIukjXri4PPn98RRzMzMrBp1mZxExGMR8ZuIuD8i\n7oqIA4E7gandPfZ558EKK3gyNjMzs1opRJ+THnIP8MmONpo6dSqjRo1apKyxsZHGxkYAxo9Pdyh+\n6CHYeONeidPMzKywmpqaaGpqWqRszpw5fRqDIjo7OKZvSGoFvhgR07q439+BNyJi1zbWTwSam5ub\nmThxYpvHmTsXll4azjoLDj20KxGYmZn1Ty0tLUyaNAlgUkS09Pb5CnFZR9JISRNyfUbWyJ6Py9af\nJOnC3PZHSdpR0pqS1pf0M+AzwFndjWWJJWDCBN+h2MzMrFaqSk4kfV7S5rnnh0t6QNJlkpap4pCb\nAPcDzaT5S04FWoATsvVjgXG57Ydl2/wbuAX4GLBVRNxSxbkX09DgOxSbmZnVSrUtJz8BlgaQ9DFS\nonAtsDpwWlcPFhG3RsSgiBhcthyQrd8/Ij6b2/4nEbF2RIyMiBUiYquI+EeVdVlMQwM8+ijMnt1T\nRzQzM7POqjY5WR14JHv8JeDqiPg/4HBg254IrJYaGtK/995b2zjMzMwGomqTk3eBJbLHWwN/zx6/\nRtaiUs/Gj4fRo93vxMzMrBaqHUp8O3CapDuAjwN7ZOXjged7IrBaGjQIbrwR1lyz1pGYmZkNPNW2\nnBwBvAfsChwWES9k5duS3SOn3k2cCGXToZiZmVkfqKrlJCKeBb5QobzbM7SamZnZwFbtUOKJ2Sid\n0vOdJP1Z0omShvVceGZmZjbQVHtZ51ek/iVIWgP4HTAX2A04pWdCMzMzs4Go2uRkPPBA9ng34B8R\nsRewH2losZmZmVlVqk1OlNt3a9IEbADPAct3NygzMzMbuKpNTu4DviPpK8CWwDVZ+erArJ4IzMzM\nzAamapOTrwMTSTfa+1FEPJGV7wrc2ROBFcVBB8Fll9U6CjMzs4Gj2qHE/ybdbK/ct4CF3YqoYB56\nCObPh732qnUkZmZmA0O1M8QCIGkSsG729JGIaOl+SMXS0ADXXtvxdmZmZtYzqp3nZEVJNwP3Amdm\ny32SbpS0Qk8GWGsNDfDEE/Df/9Y6EjMzs4Gh2j4nPweWBNaPiGUjYllgA9JN/87sqeCKYPLk9O/d\nd9c2DjMzs4Gi2uTk88CUiJheKoiIR4DDSffX6TdWXx2WX97JiZmZWV+pNjkZBCyoUL6gG8csJCld\n2nFyYmZm1jeqTSRuAs6QtHKpQNKHgdOzdV0iaQtJ0yS9IKlV0o6d2OfTkpolzZf0mKR9u3rezmpo\ngHvugdbW3jqDmZmZlVQ7WucIYBowU9JzWdk44EFg7yqON5I0Hf5vgT92tLGk1YCrgbOBvUiz1J4r\n6cWIuL6K87drhx1gxAhYsACGD+/po5uZmVletfOcPCdpIikp+GhWPB14FPgecEgXj/c34G8AktSJ\nXQ4DnoqIo7PnMyRtDkwFejw52WijtJiZmVnvq3qek4gIUiLwfjIgaQJwIF1MTqowGbihrOw60mUl\nMzMzq2P12nl1LIvfw2cWsLQkX3gxMzOrY/WanJiZmVk/1a3p62voJWBMWdkY4I2IeKe9HadOncqo\nUaMWKWtsbKSxsbFnIzQzM6tDTU1NNDU1LVI2Z86cPo1BqetIJzeWOhpJMxrYMiIGVx2Q1Ap8MSKm\ntbPNycC2ETEhV3YZMDoitmtjn4lAc3NzMxMnTqw2PDMzswGnpaWFSZMmAUzqi/vodbXlpKPUaQ5w\nUVeDkDQSWAsojdRZI+tc+1o2MugkYOWIKM1l8kvgcEk/Bs4DtgJ2BSomJmZmZlY/upScRMT+vRTH\nJsDNQGTLqVn5hcABpA6w43JxzJS0PWl0zpHA88CBEVE+gqdHPfkkXHcdTJnSm2cxMzMb2ArR5yQi\nbqWdzrmVkqKI+AcwqTfjKtfSAocfDrvsAmPH9uWZzczMBg6P1umChob0r++zY2Zm1nucnHTBuHGp\nxcTJiZmZWe9xctIFEkyeDHfdVetIzMzM+i8nJ13U0AD33gsLF9Y6EjMzs/7JyUkXNTTAW2/B9Om1\njsTMzKx/cnLSRZtski7vuN+JmZlZ73By0kVLLQXbbw9dmFjXzMzMuqAQ85zUm6uuqnUEZmZm/Zdb\nTszMzKxQnJyYmZlZoTg5MTMzs0JxcmJmZmaF4uTEzMzMCsXJiZmZmRWKk5NumDMHXnml1lGYmZn1\nL05OumGjjeCUU2odhZmZWf/i5KQbNt3U09ibmZn1NCcn3dDQAM3N8N57tY7EzMys/yhMciLpcElP\nS5on6S5Jm7az7ZaSWsuWhZJW7MuYGxpg7lx46KG+PKuZmVn/VojkRNIewKnAccDGwL+A6yQt385u\nAawNjM2WlSLi5d6ONW/iRBg8GO66qy/PamZm1r8VIjkBpgK/ioiLIuJR4FBgLnBAB/u9EhEvl5Ze\nj7LMEkvAhhu630mRRcBPfgKvvlrrSMzMrLNqnpxIGgpMAm4slUVEADcAn2hvV+ABSS9K+rukzXo3\n0soaGpycFNmJJ8LRR8PNN9c6EjMz66yaJyfA8sBgYFZZ+SzS5ZpK/gN8FfgSsAvwHHCLpI16K8i2\nNDTAY4/Bm2/29ZmtI5deCt/5DpxwAuy22+Lr582DP/6x7+MyM7P2Dal1ANWIiMeAx3JFd0lak3R5\naN/29p06dSqjRo1apKyxsZHGxsaqYtl1V9h5Z1hqqap2t15y661wwAGw337w3e9W3qapCQ48EM44\nA448sk/DMzMrrKamJpqamhYpmzNnTp/GoHQFpXayyzpzgS9FxLRc+QXAqIjYuZPHOQX4ZER8so31\nE4Hm5uZmJk6c2P3ArbAefRQ22yx1WL72Whg2rPJ2EemSz09/CmedBYcf3rdxmpnVi5aWFiZNmgQw\nKSJaevt8NW85iYgFkpqBrYBpAJKUPT+zC4faiHS5xwawWbNg221h5ZXhyivbTkwApDTD78KFcMQR\naeTVoYf2XaxmZlZZzZOTzGnABVmScg/p8swSwAUAkk4CVo6IfbPnRwFPAw8DI4CDgc8An+vzyK1Q\nvvUtmD8fbrkFRo/ueHsJTj0VWlvhsMNg0CA45JBeD9PMzNpRiOQkIq7I5jT5PjAGeADYJiJKt9Ub\nC4zL7TKMNC/KyqRLQv8GtoqIf/Rd1FZEZ54JL7wAq67a+X0kOP30lKB89aupBeXAA3svRjMza1/N\n+5z0Ffc5sY5EpMs7V10F06fDyJG1jsjMrBj6us9JEYYSmxWCBD//OdxzjxMTM7NacnJiljNoEIxt\na3YdMzPrE05Oesjf/gbrrQfvvFPrSMzMzOqbk5MesswyqZ/Cv/5V60gGhnffrXUEZmbWW5yc9JCN\nNkpzavg+O73v9ddh0iS4+OJaR2JmZr3ByUkPGT48JShOTnrXu+/CLrvAiy+m+xr1pdtugz/8oW/P\naWY2EDk56UGTJzs56U0RcNBBcOed8Oc/w/jxfXv+iy6CPfeEP/2pb89rZjbQODnpQQ0N8MQTadr0\n1tZaR9P/nHBCupRz4YWwxRZ9f/5zzoEvfQl23x3+8pe+P7+Z2UDh5KQHfeELsNVWsNtu6f4u1nMu\nuCAlJyedlFovamHIELjkknQX6t12S5O1mZlZz3Ny0oOWXhpuuAHuuCPNNGo948Yb4eCD0/Ltb9c2\nliFD4NJLYccdUyvKNdfUNh4zs/7IyUkv2Gwz2GGHWkfRfzz9NGyzDfziF2kW11obOhSammD77VPn\n3L/+tdYRmZn1L05OrPAOOihdQhk6tNaRfGDoULj8cvj8552cmJn1tELclXigiYCddoJNNoGvfS1N\n4GbtK0KLSblhw+D3v0+XeszMrOe45aQGFiyA1VZLnTtXXRWOOQZefrl28bS2wn33wfe/D5/8JLzx\nRvvbT5sGDzzgEUmQEpRB/isyM+tR/m+1BoYNgzPPhJkz4bDD4KyzUrLy9a/DCy/0TQxvvJEmFDvg\nAPjwh2HTTeHUU9Pj2bPb3i8CpkyBjTeGFVeEXXeFs8+GGTPSOjMzs+5yclJDY8bAj38MzzwDRx+d\n5u9YYw34xjd675zvvQef+xwst1xKLO6+G77yFbj5Znj1VbjiClhllbb3l+DJJ+GWW+Dww2HWLDjq\nKPjoR+EjH4G99073GLKUrH33uykRveKKNMPs44/Dm286kTMza4+vlhfAssvC8cenpOScc3r3pnZD\nhsCGG6a5OrbbLrXYdNXw4bDllmk54QR46600fPqmm9JSrZtvTi0yo0dXf4wimT8/TRr30kuL3616\niSVg7Fj47W/h05+uSXhmZoVVmORE0uHA/wBjgX8BX4uIe9vZ/tPAqcD6wLPAjyLiwj4ItdcsvXTn\n5/FoamqisbFxsfIXXkiXZtpz6qlVBNeOJZdMQ3232aZz28+YASuskJKykhNOaOLkkxs56ig4+eSe\nja8WSu/PzJmplWTOnJSklC8rr9z+cX79azj22JTI5JeVVkqv4ahRac6V9tx5Zzr/oEGp5WvQoEUf\njxuXWuzacumlTWy5ZSPLLgsf+lAxOyd3RVt/O/WqP9WnP9UF+l99+lRE1HwB9gDmA/sAHwV+BbwG\nLN/G9qsBbwGnAOsAhwMLgM+1c46JQDQ3N0d/sMMOO0RExIIFEbfeGnH00REbbBABEU89VePgOvCp\nT0VIERMnRvzP/0Q0NUUMG7ZDbL55xLx5tY6uZ5Ten+5qaYn40Y8ijjgiYtddIzbfPGKttSKWXDK9\n1+PGdXyMTTdN27a1fPOb7e+/9dY7vL/t8OERK60Usd56KZYdd4zYd9/if+byeuq9KYr+VJ/+VJeI\n/lWf5ubmAAKYGH2QFxSl5WQq8KuIuAhA0qHA9sABpASk3GHAUxFxdPZ8hqTNs+Nc3wfx1tRPf5o6\n0+6xB1x3XfpVvOKK6TLN976XHhfZRRelSzg33QSXXZbqM3JkupnfiBG1jq5YNt44LZXMn9+5EVPX\nXJMuFba2phSjtXXRxx1dRhs2LM3l8tpriy+vv5760Sxc2P4xTjoJfvKT1Fq2zDLp3/zj1VeHAw9s\n/xhXXpk2QxlrAAAQjUlEQVQ+67Bo603p8cSJMGFC2/u//jpcfTU8/3waAj50aFqGDfvg3403Tq1D\nbVmwIJ1v8OD6b0EyK7KaJyeShgKTgBNLZRERkm4APtHGbpOBG8rKrgNO75UgC2TBgtS58sEH0xf5\n1KkpKZk0qX6GtK66Kuy3X1oi4Kmn4MgjUydd67zOJnIrrNC98wwenCab647PfCYdJ5/YvPwyPPpo\nerzKKh0nJ8ceC4891vb6H/yg/eTkuedgn33S4913r7zNo4/COuu0fYzjj4cTs/+p8klN6d+PfQyu\nvbbdarDbbimW0mU1adHHBx2UOpa35amnUif00vb33ptmKi4dR0qdsFdaqe1j/OEPKVHL75OPYdy4\n9Hq359vfhhdfTAnuwoUfLKXne++dfkC1ZcYMaGxcdP/nnkt94kqv5+WXt99Bf9o0uP76D96D/Psx\ndGh6Dfbaq/163HprSvTLL3uWltVWS5392zJ/fprFutK+8+alS+1jxng+pK4qwsu1PDAYmFVWPot0\nyaaSsW1sv7Sk4RHxToV9+oWhQ9MIm+237/g/wXogwZprpi8u678mT05Ld8yYkf7Nj3TqyqinDTZI\nXyS77JJa7N59NyX7CxZ88LijDuK77gprr/3B9uXH6EwiuPbaqX9Z6cJaqRWr9HippTo+xqBBaftS\nMjB//qIX6zpqUXv11ZTolZ+/tJRaqNrz7LMpORk0KP39Dh686OOOWpZGjkyfifx+f/lL6iBeel07\nSsBnzkzJRfl7UVo22KDj5OSAA1LC15YTT0xzUbXlkUfSj8O2fOQjqXVxrbXa3ubYY1PL4pAhH7x+\ngwd/8HzChI5not5ll/R+tJX0HnJI+6/Fk0+m6SzK9y89X3PN9s/f04qQnPSVEQDT+8k417ffnkNL\nS0utw+gxc+a4PkXVn+oC8NZbc3jyycr1efjhjvffcMP213f0Uu26a8fn6OgYxx33weOpU+fwwx8u\nusOsWWlpy6abpqU7MXzrW+2v78wxDjpo0efNzXPYb78Pdnr++bS0ZfPN09KdGH75yzTFQv5SZ/7y\n5+jR7R9j7lw477zF92tthXPOmcPBB7cwa1b7k1uOH59Ga+ZbkfKPl1uu43qMGpW2LSXs5UnnrFnt\nH+O55z5ISvOvQ+n5kCHvf3f2ycV3RY0nXMgu68wFvhQR03LlFwCjImLnCvvcCjRHxDdyZfsBp0dE\nxcngJe0FXNqz0ZuZmQ0oX46Iy3r7JDVvOYmIBZKaga2AaQCSlD0/s43d/glsW1b2/7LytlwHfBmY\nSRoZZGZmZp0zgjRS9rq+OFnNW04AJO0OXAAcCtxDGnWzK/DRiHhF0knAyhGxb7b9asCDwNnAeaRE\n5mfAdhFR3lHWzMzM6kjNW04AIuIKScsD3wfGAA8A20TEK9kmY4Fxue1nStqeNDrnSOB54EAnJmZm\nZvWvEC0nZmZmZiV1MjOGmZmZDRQDIjmRdLikpyXNk3SXpA4G0fVJTFtImibpBUmtkha7Q4qk70t6\nUdJcSddLWqts/XBJv5D0qqQ3JV0pacWybZaRdKmkOZJel3SupJE9XJdjJN0j6Q1JsyT9SdL4Oq7P\noZL+lZ1jjqQ7JX2+bJu6qEuFuv1v9nk7rR7rI+m4LP788kg91iV3rpUlXZzFMzf77E2sxzop/T9b\n/v60Svp5HdZlkKQfSHoqi/UJSd+psF1d1Cc7z5KSfiZpZhbv7ZI2KWR9+mKO/FoudPG+PX0Y1+dJ\nfWx2AhYCO5at/3YW5xeADYA/A08Cw3LbnEMafbQlsDFwJ3Bb2XH+CrQAmwCbAY8Bl/RwXa4FvgKs\nC3wMuDqL60N1Wp/ts/dnTWAt4IfAO8C69VaXsvNtCjwF3A+cVqfvzXHAv4EVgBWzZdl6rEt2ntHA\n08C5pJmyVwW2BlavxzoBy+XelxVJgxUWAlvUYV3+D3iZ9H/BKsAuwBvAEfX43mTnuZw0mOSTwBrZ\n39NsYKWi1adHK17EBbgLOCP3XKQOtEfXOrZcTK0snpy8CEzNPV8amAfsnnv+DrBzbpt1smN9PHu+\nbvZ849w22wDvAWN7sT7LZ+fdvD/UJzvPf4H967UuwJLADOCzwM0smpzUTX1I/5m2tLO+buqSHfdk\n4NYOtqmrOpXF/jPgsXqsC3AV8JuysiuBi+q0PiNIN8j9fFn5fcD3i1affn1ZRx/ct+fGUlmkV6q9\n+/bUnKTVSSOU8nG/AdzNB3FvQhptld9mBvBsbpvJwOsRcX/u8DeQ7izZ0Fvxk34NBikDr+v6ZE27\newJLAHfWcV1+AVwVETflC+u0PmsrXQ59UtIlksbVcV12AO6TdIXSJdEWSe/Pm1qndSrFPpQ0t9Rv\n67QudwJbSVo7i38CqcXh2jqtzxDSrWLKb+8yD9i8aPUpxFDiXlTNfXuKYCzpjawU99js8Rjg3ezD\n09Y2Y0nNku+LiIWSXstt06MkifRr6faIKPUFqLv6SNqANKnfCOBN0i+FGZI+Qf3VZU9gI9J/LOXq\n7b25C9iP1Aq0EnA88I/s/aq3ukBqWj8MOBX4EfBx4ExJ70TExdRnnUp2BkYBF+ZiqKe6nExqKXhU\n0kJSH81jI+J3uTjqpj4R8ZakfwLflfRoFsNepKTicQpWn/6enFjfOxtYj/QLo549Ckwg/ee6K3CR\npE/VNqSuk/QRUrK4dUQsqHU83RUR+dkpH5J0D/AMsDvpPas3g4B7IuK72fN/ZYnWocDFtQurRxwA\n/DUiXqp1IFXag/TlvSfwCCnBP0PSi1niWI/2Jk1c+gLpMksLcBnpCkOh9OvLOsCrpM5YY8rKxwBF\n/oN5idQ3pr24XwKGSVq6g23Ke1EPBpalF+ov6SxgO+DTEfGf3Kq6q09EvBcRT0XE/RFxLPAv4Cjq\nry6TSJ1HWyQtkLSA1JHtKEnvkn7x1FN9FhERc0id7dai/t4bgP8A5XcjnU7qgFmKpd7qhKRVSB17\nf5Mrrre6nAKcHBG/j4iHI+JS0sSfpXsU11t9iIinI+IzwEhgXERMBoaROsoXqj79OjnJfimW7tsD\nLHLfnjtrFVdHIuJp0puYj3tp0vW6UtzNpMw3v806pP/USvcY+icwWtLGucNvRfoA3t2TMWeJyU7A\nZyLi2XqvTwWDgOF1WJcbSCOoNiK1BE0gdYC7BJgQEaX/lOqlPouQtCQpMXmxDt8bgDtY/BLzOqTW\noHr+2zmAlPheWyqow7osQfpxm9dK9r1Zh/V5X0TMi4hZkpYhdVb9c+Hq01M9gYu6kJp757LoUOL/\nAivUOK6RpC+KjUgf+K9nz8dl64/O4tyB9OXyZ9J1wfyQrrNJwxA/TfqFfAeLD+m6lvRltCnpUssM\n4OIersvZwOvAFqQMurSMyG1TT/U5MavLqqThdCeR/iA/W291aaN+5aN16qY+wE+AT2XvzWbA9aQv\nweXqrS7ZeTYhdVA8hjR0fS9SH6c96/H9yc4j0lDTH1VYVzd1Ac4ndfTcLvu87UzqS3FiPdYnO8//\nIyUjqwGfI00rcAcwuGj16dGKF3UBpmR/LPNIWd0mBYhpS1JSsrBsOS+3zfGkoV1zSXeCXKvsGMOB\nn5MuX70J/B5YsWyb0aRfyXNICcRvgCV6uC6V6rEQ2Kdsu3qpz7mkZs55pF8SfydLTOqtLm3U7yZy\nyUk91QdoIk0FMI/0xXEZuTlB6qkuuXNtR5q7ZS7wMHBAhW3qpk6kL72F5THWW11IPyBPI30Rv036\nkj4BGFKP9cnOsxvwRPb38wJwBrBUEevje+uYmZlZofTrPidmZmZWf5ycmJmZWaE4OTEzM7NCcXJi\nZmZmheLkxMzMzArFyYmZmZkVipMTMzMzKxQnJ2ZmZlYoTk7MzMysUJycmNU5STdLOq0L268qqVXS\nhtnzLbPn5Xca7XWSzpf0x74+b7UkHSfp/lrHYdbfOTkxKxhJF2TJwtkV1v0iW3dernhn4LtdOMWz\nwFjgoVxZt+9j0dUkqY75nh9mvczJiVnxBCmB2FPS8FJh9rgReGaRjSNmR8TbnT548nJEtPZUwNY9\nkobUOgazInFyYlZM9wPPAbvkynYhJSaLXFYob7GQ9LSkYyT9VtIbkp6RdHBu/SKXdXI2l/QvSfMk\n/VPS+rl9lpV0maTnJb0t6d+S9sytP590p+2jsmMvlLRKtm59SVdJmpPFc6uk1cvq8E1JL0p6VdJZ\nkga39cKULq1I2jur62xJTZJGlr0GR5btd7+k7+Wet0o6JIvtbUmPSJosac3sNX1L0h3lsWb7HiLp\n2Wy/yyUtVbb+oOx487J/D6vw+u8u6RZJc4G92qqv2UDk5MSsmAI4DzggV3YAcD6gTuz/DeBeYCPg\nbOAcSWuXHT9PwCnAVGAT4BVgWi5JGAHcB2wLrA/8CrhI0ibZ+qOAf5JujT4GWAl4TtLKwK2kW7R/\nGtg42ybfUvBZYI1s/T7AftnSnjWBnYDtgO1JidH/drBPJd8BLgAmANOBy4BfAj8CJpFel7PK9lmb\ndOv57YFtSHV6/xKcpC+Tbjt/DPBR4P+A70v6StlxTgJOB9Yl3ZrezDJuSjQrrkuBkyWNI/2Q2AzY\nA/hMJ/a9JiJ+mT3+saSp2X6PZ2WVEpzjI+ImAEn7As+T+rNcGREvAvn+JL+Q9Hlgd+C+iHhD0rvA\n3Ih4pbSRpCOA2UBjRCzMip8sO+9rwBEREcBjkq4BtgJ+2079BOwbEXOz81yc7dOVvjcA50XEH7Jj\nnEJKsE6IiBuysjNISWLecOArEfFSts3XgGskfTMiXiYlJt+MiL9k2z+TtUIdClycO87puW3MLMfJ\niVlBRcSrkq4G9id9GV8TEa9JnWk44cGy5y8BK7Z3OuCu3LlflzSD9KseSYOAY0ktBh8GhmVLR31d\nJgC35RKTSh7OEpOS/wAbdHDcmaXEJLdPe/VrS/51mpX9+1BZ2QhJS0bEW1nZs6XEJPNPUvK4jqS3\nSK06v5V0bm6bwaQkLa+5injNBgQnJ2bFdj7pskIAU7qw34Ky50H3LuMeDXyNdPnmIVJScgYpQWnP\nvE4cu5pYO9qnlcVbh4Z2cJxop6yzr92S2b8HAfeUrStP0DrdidlsoHGfE7Ni+xspARgC/L0XzyNg\n8vtPpGWA8cAjWdFmwF8ioikiHgSeztbnvUtqIcj7N7BFex1ce8krpH4vAGRzuCzWsbWCzgwTXkXS\n2NzzT5ASj0ezyzovAmtGxFNlS36UlYcjm7XDyYlZgWXDfT8KrF926aM3fE/SZyVtQOok+gpQ6hPx\nOPA5SZ+QtC6pQ+yYsv1nAg3ZaJTlsrKzgKWByyVNkrRWNspmbXrXTcBXJG0u6WNZfd7rxH6VrpmV\nl70DXChpQ0lbkFqQLs/1tTkOOEbS1yStLWkDSftJ+noH5zGzjJMTs4KLiLdy/R0qbtLB885sE6TR\nLmeQRvmsAOwQEaUv9B8CLaSWnJtIfTz+VHaMn5JaEB4BXpa0SkS8RhqNMxK4hTTi5yAWvyzT004i\njRK6Klv+xOIdcTvzOlUqexz4I3At6fV4ADj8/Y0jfkuq4/6klqNbgH1JrU3tncfMMur9H2NmZmZm\nneeWEzMzMysUJydmZmZWKE5OzMzMrFCcnJiZmVmhODkxMzOzQnFyYmZmZoXi5MTMzMwKxcmJmZmZ\nFYqTEzMzMysUJydmZmZWKE5OzMzMrFCcnJiZmVmh/H9o3NnoLofEpgAAAABJRU5ErkJggg==\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAADeCAYAAADmUqAlAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzt3Xm8nOP9//HXO4ukglhCQu0EQS2JxF5StVdVF0RrbWlK\nW9L+vr5F1dKFalFaa+2lQVuUUtT6tVRDjqX2LaGWhDQESWT9/P647pHJZGbOzGTmnJlz3s/HYx7n\nzHVvn2vWz1zXdV+3IgIzMzOzZtGjswMwMzMzy+fkxMzMzJqKkxMzMzNrKk5OzMzMrKk4OTEzM7Om\n4uTEzMzMmoqTEzMzM2sqTk7MzMysqTg5MTMzs6bi5MSqImm+pJ/UuO1ESZfl3T8429/Q+kVYO0lr\nZPH8oLNj6c4k3SfpqTrvc6HXXjMr8j7ZIXtdfraOx6j5fWzWEZycdEN5ScF8SduUWOc/2fKbCxZF\ndqvF/CLb1vX6CZJGSTq6nvu09mWvlXPrtLtGXFOjon3mvS/mS5on6U1Jd0jaoQExlVIs1qofE0m7\nSzqpzDE6/Nolkk4qeIwLH++VOjoma069OjsA61QzgQOAh/MLsw/iTwMfF9nmU8DcGo+3PilBaaQD\ngI2Acxp8HOu67gSuAgSsBRwJ3CNpj4i4o6ODiYj7JX0qImZXuekepNhPKbJscd7HiyuA0cD0Isve\n7+BYrEk5OenebgO+Jun7EZGfNBwAPAYMKNyghg/I/G3n1Lptq5K0ZETM6Ow4rCovRsQfc3ck3QQ8\nBRwDFE1OJAlYIiJmNSKgGt93qvP+6ukvETG1mg0k9QFmR5Gr1dbjfeb3anNxt073FcBYYAVg51yh\npN7AV4E/UuTDrbCvWtLJWdk6kq6Q9J6k9yVdJqlvwbal+v37SbpI0hRJ0yRdKWnZgm2/KOlvWTP7\nx5JelvRjST3y1rkX2BPIjR2ZL+nVvOV9snhfkDRT0luS/iJprSL1PDw7xseSxknaor0HNK+77LOS\nzpc0GfhPtuwKSROKbHOypPkFZfMlnStpb0n/zmJ4WtKu7Rx/JUlzJJ1YZNl62X6PzO73yprYX8we\niymSHpC0U3v1rFUlz2HB+kMlPSRphqRXJX27yDpLSDpF0kvZPl+X9EtJS9Qr7oh4GphCakXJHTf3\nHB0g6WlSK+Ou2TJJOiZ7zmZKmiTpwsLXdLbuj5W6UKdLulvShkXWKTrmRNKWkm6TNFXSR5KelPS9\nbNnlpFaT/K6qeQXx/6Rgf5tL+nv2HvxQ0l2StixYJ/ca30bSWZLeyY59g6QVqn5wS8ir836Sfibp\nDVJLy9KSDin1PquhHkX3YZ3PLSfd20TgEWAUC34R7gEsA1wLVDJ2I/cr5nrgVeBHwFDgW8Bk4Lgi\n6+YT8DvgPeAkUtfPkcDqwMi89Q4BPgTOBD4CPgecCiwN/G+2zs+A/qQuqWOyfX8EkH0B3prtcyzw\nm2zbnYGNgfzE4evAUsCFWcz/C/xF0toRMY/2nQ+8Q2pOXzKv7qXGEhQr3x74cravD4HvA3+WtHpE\nvFfsoBHxjqT7gX2BnxYs3p/UjH99dv8U0nN1MfAo6TnfgvTc3d1+FWtyCO0/hznLk56v60mJ8r7A\nBZJmRcQV8ElrxS3ANsBFwPPAZ4AxwGDS47fYJC0HLAe8VLBopyyu35GSl4lZ+cXAQcBlpO7FtYDv\nAZtJ2jb3GpL0U+AE4G/A30mP/Z1A7yJhLPQakbQzqe5vkV7Lk4AhwBeA35Iej1WAz5NezyVbUbL9\nbQj8HzANOJ30Wvk2cJ+kz0bEowWb/BaYCpwMrEl6zH9H+iypxArZ85dvbkRMKyg7EZgF/AroA8xm\nwWOR/z7rl9Vjoyrrscg+rElEhG/d7AYcDMwjfRgeSern7ZMtuw64K/t/AnBzwbbzgZ/k3T8pK7u4\nYL2/AO8UlE0ALiuIYz7wL6BnXvn/y+L7Ql5ZnyL1uID0Zdc7r+wW4NUi6x6aHev7ZR6XNbJ13gGW\nySvfK4tnjwoe1/nAfYAKll1eIq6TgHlFHuOZwJp5ZZ/Jyo9sJ4bDs1g3LCh/GvhH3v3HC5/bxXxN\nzQfObWedSp/De7M6HJ1X1htoA97OvVaAbwBzgK0L9nlEtv1WpV577dTjYlKL4gBgBHBXkXjmZ8de\nv2D77bJl+xWU75yV75/dH0BqbflrwXo/y9bLf5/skB3/s9n9HqQfAq8AS5epy28LX1sF8ee/j2/M\nXnNr5JUNIn3J31vkNX57wf7OJCUOJePJe73PL3F7tqDO80kJ4RJVvM+qrcci+/CtOW7u1rHrSb/u\nvyBpKdIvr2uq3EeQfqnle4D062ipCra/OBZukbiALBn45AB5ffmSlsqakB/MYt+ggmN8GXiX9Ouu\nPddGxAd59x8g/fJcu4JtA/h9ZJ+Ai+EfETHxk51G/Bv4oIIYbiA9dvvlCrJfkxuSWsNy3gc2krTu\nYsZZsSqfw7mkJCG37RzSa2wlYFhW/FXgOeBFSSvkbqTkRizc8laNb5JeK++QWha3Bs6MiMJB1vdF\nxAsFZV8lPbZ3F8T0OKm1KBfTzqSE67cF2/+mgvg2J7VW/CYiPqysSqVlrYo7AzdGxGu58oiYRGq1\n2q7gfRzkPTeZB4CepAS/PQHsQ2rVyb8dWmTdK6L4+JhF3mc11qMe71VrAHfrdHMRMUXSXaRBsP1I\nv8r+XMOuXi+4n+t6WI6sa6VUCMDLBTFNl/Q26QMY+KTZ+eekD/dlCrbvX0F86wAvxMIDf0tZqO85\nIt7PWqCXq2BbWNC8vziK9X+/114MEfFfSXeTuhtyp5HuT/qVf2Peqj8BbiJ9sT8N3A78IUuCGqLK\n5/CtiJhZUPYiKelYExhH6rrZgJRIFApSIlOLv5KS2CC16jxTJBYo/jwPBpYlJTblYlo9+1v42p8i\nqWi3XZ51sn090856lVqRlCC+WGTZc6TPhNWy/3MKX5/57/dKPBCVDYidWMWyWupRbv/WiZycGKRf\nFb8HVgb+XuOvsVJjMcr2dVdCUn9SP/L7wI9JTdofk35Bn079B3Yvbl2KfZGV+nXWswExXAtcJmmT\niHgK+Bpwd/6XQUQ8IGkdYG9gF1JrwRhJ346Iuk9W1qDnsAfwb9J4h2KPS60DHN+IiHsqWK/Y89yD\nNNbqgBIxFUukWlHD3u8Fij3GlSyrx/6tEzk5MUi/qC8CtiSvO6CDiPRr8/5PCqR+pETp1qxoR9Iv\nsr0j4qG89dYpsr9SScArwAhJPaOyQa319h7pF3WhNRtwrJtIz+d+2aDD9UgtFguJiPeBK4ErJS1J\napo/mTSQs952pPLnEGAVpbk98r881ic9v7nBy68Am0TEvQ2It1avkAbKPhzlTyvOdTsMJu/Xu6QB\ntN/68ArpfbMxUC6JqrS74l1gBunxLTSENDajFc5k6Sr1MHwqsZG6UUiTIp1MGlDa0Y6QlJ8oH0lq\nUbgtuz+P9GGcf9rwEtl6haZTvJvnL6Rm3+/WI+AavAL0l7RxrkDSysCX6n2gSGc83EHq2tmfdLbD\nX/PXkbR8wTYzSF0MffLWWUbS+pLyu2BqVc1zCOmH0+i8dXuTzrp4lzQwFtJ4qVUlHV64saS+WcLV\n0a4nxb7I1PCSemYtSJAG2c4lncWTb0wFx2gjJWjH5O2vmOnZccs+f1lX553A3pJy3U1IGkg6++aB\niCjXNdsUuko9LHHLSfe1UPNrRPyhswIBliANILyeNIbgO6QPkr9lyx8mtTxcpQVTpH+D4r8MxwP7\nSjqTdIrsR9l+riKd3nlWNufBA6TThXcCzouIeiVlpZq1rwV+CdyU1aEf6cv3BdJZU/V2HXA16cv/\njoIBvgDPSrqP9HhNBYaTBnPmT0G/D+kso0NIj197tpB0QpHye6nuOYR0Vs6xktYkjSHYH9gEODyv\n5esPLDjFeCTwECmpHULqytqFBYlMh4iI/5N0EfAjSZuRviznkFqvvko6JfyGbGzJr7P1/kZKxDcH\ndqN4188nr6uICEnfAW4GnlCa0+Rt0ntnw4jYPVt1fLbdbyXdQTpz57oSof+YNCj1IUnnk5LJI0jv\nzWNLxVJhebH1viapWKJwZ0RU0vVV6lj1qIc1AScn3VclTb7F5uBYnGtylNrfd0lzMZxCOoPhGvLm\nWImIqZL2JJ2u+FPSl9wfSE3ahTN2ng9sSvpCPYbUfP63iJgvaXfSvBIHkM7e+S8pSckfBFrtfCTF\n1lu0MNXhS8BZpCRlAmmekfVYNDlZ3BggfXHNJCVB1xZZfg7wRdLZDX1Ij9PxwK+LHLMSQeoW3LLI\nshMj4uEqnkNIz80hpIGpuTlzjsofD5N9Se9Nam04iNQKNYM0nuVsFh4YWc3zt1jrRcR3JD1Gaun5\nOamFZCIpwXsob70TJM0kJak7ks4M2oXUnVnsfZJ/jDuzhOwk4AekFqlXWPgsmhtIyeb+LJjrJJec\nLBR/RDwraXvgNNLrskcWzwER8Vi5WCooL7be+SWWjWRBclZuf6Ue+3rUw5qAfBaVmZmZNZOmGHMi\naXtJNytNaz1f0hcr2GZHSeOVpqx+UdLBHRGrmZmZNVZTJCekpucnSP3j7TblZP3QfyNNs70pqYn6\nkmxKZzMzM2thVXXrZGdUHE+aWvmNhgSULoL2pYi4ucw6vwR2j4hN8srGAv0jYo9S25mZmVnzq6rl\nJCLmAv9D5w+k3Yp0Kl6+O0jTTJuZmVkLq6Vb5x7SRZk60yDS6P18k4FlJPUpsr6ZmZm1iFpaQP4O\nnC7pM6Tz6KfnLyzXHdOZsotv7Uo6pe/jzo3GzMyspfQlzWh9R0T8t9EHqyU5yZ2f/oMiy4LS1wqp\np0nAwIKygcAHZaaM3pXqr7ZrZmZmC3yddD22hqo6OYmIZjjD55/A7gVlu2TlpUwEuPrqqxkyZEiD\nwuo4Y8aM4eyzz+7sMOrG9WleXaku4Po0s65UF+ha9Xnuuef4xje+AR10JefOHtgKfHKht3VZMJ3w\n2pI2BaZGxH8knQasEhG5uUwuBI7Kztq5jDQF+VeBcmfqfAwwZMgQhg5txGzhHat///5doh45rk/z\n6kp1AdenmXWlukDXq0+mQ4ZF1NQKImkHSbdIejm73ZxNGVyrLYDHSWNYgjTFdRtpOnNIA2BXy60c\nEROBPUnXUHiCNH31NyOi8AweMzMzazFVt5xI+gbpYmC56zYAbEu6cNshEVF1X1RE3E+ZRCkiDi1S\n9n/AsGqPZWZmZs2tlm6dE4BjIyK/I+1cST8ATqQDBsqYmZlZ11VLt87aQLHLy98MrLV44VilRo0a\n1dkh1JXr07y6Ul3A9WlmXaku0PXq05GqviqxpJeBX0XERQXlo4EfRsTgOsZXN5KGAuPHjx/fFQco\nmZmZNUxbWxvDhg0DGBYRbY0+Xi3dOmeSunE2Ax7OyrYFDgGOrlNcZmZm1k3VMs/JBZImAT8E9s2K\nnwP2i4i/1jM4MzMz636qSk4k9SS1ktwbETc2JiQzMzPrzqq9KvE84E5gucaE0wHmzu3sCMzMzKyM\nWs7WeZp0xk5revnlzo7AzMzMyqglOfkx8GtJX5C0sqRl8m/1DrDunnmmsyMwMzOzMmo5W+e27O/N\npKnmc0THXZW4dk5OzMzMmlotycnIukfRkZycmJmZNbVqz9bpBewAXBYRbzQmpAYbORIiQGp/XTMz\nM+tw1Z6tMxf4H2prcWkOo0c7MTEzM2titQyIvYfUemJmZmZWd7W0gPwdOF3SZ4DxwPT8hRFxcz0C\nMzMzs+6pluTk/OzvD4osa/6zdczMzKyp1XJtnVq6gszMzMwq0jSJhqSjJE2QNFPSI5KGt7P+1yU9\nIWm6pLckXSpp+Y6K18zMzBqj4uRE0m2S+ufd/5GkZfPuryDp2VqCkLQfcCZwErA58CRwh6QBJdbf\nFrgS+D2wIfBVYARwcS3HNzMzs+ZRTcvJrkCfvPvHA/ktFb2A9WuMYwxwUURcFRHPA6OBGcBhJdbf\nCpgQEedFxGsR8TBwESlBad8rr8Dtt9cYqpmZmTVSNclJ4eQgdZksRFJvYBhwd64sIgK4C9i6xGb/\nBFaTtHu2j4HA14BbKzroVVfBgQemydjMzMysqTTDmJMBpDN8JheUTwYGFdsgayn5BnCdpNnA28B7\nwHcrOuKIETBlCrz2Wq0xm5mZWYNUk5wEC1/ojyL3O4SkDYFzgJOBoaQup7VIXTvtG56NtR03rhHh\nmZmZ2WKo5lRiAVdImpXd7wtcKCk3CVuf4pu1awowDxhYUD4QmFRimx8BD0XEWdn9pyUdCTwg6YSI\nKGyF+cSYMWPo378/fOpTcPzxcPXVjBo1ilGjRtUYvpmZWdcxduxYxo4du1DZtGnTOjQGRYXjLiRd\nXsl6EXFo1UFIjwD/ioijs/sCXgfOjYhfFVn/z8DsiDggr2xr4EHg0xGxSFIjaSgwfvz48QwdOhS+\n9jV45x24//5qwzUzM+tW2traGDZsGMCwiGhr9PEqbjmpJemowlmkVpnxwDjS2TtLAlcASDoNWCUi\nDs7WvwW4WNJo4A5gFeBsUoJTqrVlYSNGwCmnwLx50NOT2pqZmTWLpri6cERcn81pciqpO+cJYNeI\neDdbZRCwWt76V0paCjgK+DXwPulsnx9VfNARI2D6dHjuOdh44/pUxMzMzBZbUyQnABFxPguu21O4\nbJFWm4g4Dziv5gMOHQoDB8Kbbzo5MTMzayJNk5x0uKWXhrffBtVluhYzMzOrk2aY56TzODExMzNr\nOt07OTEzM7OmU1O3jqTBwEhgJQoSnIg4tQ5xmZmZWTdVdXIi6XDgAtLkaZNYeJbYIJ1xY2ZmZlaT\nWlpOfgycEBG/rHcwZmZmZrWMOVkO+FO9AzEzMzOD2pKTPwG71DuQTjV7NsyZ09lRmJmZGbV167wM\n/FTSVsC/gYW+1SPi3HoE1mFefx3WWw9uuQV23rmzozEzM+v2aklOjgA+AnbIbvkCaK3kZNVVoW9f\nGDfOyYmZmVkTqDo5iYi1GhFIp+nRA7bYAh59tLMjMTMzMxZzEjZl6hVMpxkxIrWcmJmZWaerKTmR\ndJCkfwMzgZmSnpJ0YH1D60AjRqTr7Lz5ZmdHYmZm1u1VnZxI+gFpErbbgH2z2+3AhZLG1De8DjJ8\nePrr1hMzM7NOV0vLyfeA70TE/0bEzdntWOBI4Pv1Da+DfPrTsMoqHndiZmbWBGpJTlYGHi5S/nC2\nrDUNH+6WEzMzsyZQ6zwn+wK/KCjfD3hpsSPqLGecAf36dXYUZmZm3V4tyclJwHWSPgs8lJVtC+xE\nSlpqIuko4P8Bg4Ange9FRMl+FklLZLF8PdvmLeDUiLiipgDWW6+mzczMzKy+apnn5C+StgTGAF/K\nip8DRkTE47UEIWk/4EzSBG/jsn3fIWm9iJhSYrM/ASsChwKvkLqUFuvUaDMzM+t8tbScEBHjgW/U\nMY4xwEURcRWApNHAnsBhwBmFK0vaDdgeWDsi3s+KX69jPGZmZtZJKmppkLRM/v/lbtUGIKk3MAy4\nO1cWEQHcBWxdYrO9gMeA/5X0hqQXJP1KUt9qj29mZmbNpdKWk/ckrRwR7wDvk66hU0hZec8qYxiQ\nbTO5oHwysH6JbdYmtZx8TOpaGkCae2V54JtVHt/MzMyaSKXJyeeAqdn/IxsUSzV6APOBAyLiI/hk\ncrg/SToyImZ1anRmZmZWs4qSk4i4P+/uBOA/WdfLJ7Jr7KxWQwxTgHnAwILygcCkEtu8DbyZS0wy\nz5Fab1YlDZAtasyYMfTv33+hslGjRjFq1KgqwzYzM+t6xo4dy9ixYxcqmzZtWofGoIIco/0NpHlA\nrosnv3wF4J2IqLZbB0mPAP+KiKOz+yINcD03In5VZP3DgbOBlSJiRla2N/BnYKliLSeShgLjx48f\nz9ChQ0sHc8YZMGsWnHhitdUwMzPrktra2hg2bBjAsIhoa/Txajn1Nje2pNBSpDEgtTgLODy7oOAG\nwIXAksAVAJJOk3Rl3vp/BP4LXC5pSDbnyhnApYvdpTNhAlx77WLtwszMzGpX8anEks7K/g3gp5Jm\n5C3uCWwJPFFLEBFxvaQBwKmk7pwngF0j4t1slUHkdRlFxHRJOwO/BR4lJSrXAYvf3DFiBFx0EXz4\nISy99GLvzszMzKpTzTwnm2d/BXwGmJ23bDZpVtdf1xpIRJwPnF9i2aFFyl4Edq31eCUNHw4RMH48\n7Lhj3XdvZmZm5VWcnETESABJlwNHR8QHDYuqMw0Zkq6xM26ckxMzM7NOUMuYk2MoktRIWr6WSdia\nTs+eMGwYPFrysj5mZmbWQLUkJ9dS/AJ/+2bLWt+IEanlxMzMzDpcLcnJlsC9Rcrvy5a1vhEj4PXX\nYXLhpLVmZmbWaLUkJ32AJYqU9wY+tXjhNIltt/U8J2ZmZp2kluRkHHBEkfLRwPjFC6dJrLIKnHoq\nDCyctNbMzMwarZpTiXN+DNwlaVMWXEl4J2A4sEu9AjMzM7PuqeqWk4h4CNga+A9pEOxewMvAJhHx\nQH3DMzMzs+6mlpYTIuIJ4Ot1jsXMzMyssuRE0jK5Sdfam8uky07OZmZmZh2i0paT9yTlrkT8PsUv\n/Je7IGDVVyU2MzMzy6k0OfkcMDX7f2SDYjEzMzOrLDmJiPuL/d/lTZoEN98Mhx0GvWoanmNmZmZV\nqnTMySaV7jAinqo9nCbz0kvw7W/DllvCppt2djRmZmbdQqXNAU+QxpPkxpWU03XGnAwdCj16pOvs\nODkxMzPrEJXOc7IWsHb29yvABOBIYPPsdiTwSras6+jXDzbe2FcoNjMz60CVjjl5Lfe/pD8B34+I\n2/JWeUrSf4CfAjfVN8RONny4r1BsZmbWgWq5ts5nSC0nhSYAG9YaiKSjJE2QNFPSI5KGV7jdtpLm\nSGqr9dhljRgBTz8NM2Y0ZPdmZma2sFqSk+eA4yR9cmXi7P/jsmVVk7QfcCZwEqmb6EngDkkD2tmu\nP3AlcFctx63I8OEwbx48/njDDmFmZmYL1JKcjAZ2Bd6QdJeku4A3srLRNcYxBrgoIq6KiOez/cwA\nDmtnuwuBa4BHajxu+zbeGPr2ddeOmZlZB6nlwn/jSINjfww8ld1OANbOllVFUm9gGAuucExEBKk1\nZOsy2x1KGqB7SrXHrErv3rDrrqn1xMzMzBqu1gv/TQcurlMMA0inH08uKJ8MrF9sA0mDgV8A20XE\nfEl1CqWEm7rWGF8zM7NmVku3DpIOlPSgpLckrZGVjZG0d33DK3rsHqSunJMi4pVccaOPa2ZmZh2j\n6pYTSd8BTgV+Q+rayU269h5wDPDXKnc5BZgHDCwoHwhMKrL+0sAWwGaSzsvKeqTQNBvYJSLuK3Ww\nMWPG0L9//4XKRo0axahRo6oM28zMrOsZO3YsY8eOXahs2rRpHRqD0vCOKjaQngWOj4ibJH0IbBoR\nr0raGLgvIsqeYVNin48A/4qIo7P7Al4Hzo2IXxWsK2BIwS6OIl2Q8CvAxIiYWeQYQ4Hx48ePZ+jQ\nodWGaGZm1m21tbUxbNgwgGER0ZipO/LUMuZkLaDYebWzgH41xnEWcIWk8cA40tk7SwJXAEg6DVgl\nIg7OBss+m7+xpHeAjyOiplOZzczMrHnUkpxMADYDXiso340a5zmJiOuzOU1OJXXnPAHsGhHvZqsM\nAlarZd9mZmbWWmpJTs4CzpPUlzQQdYSkUaRJ2L5VayARcT5wfollh7az7Sk0+pRiMzMz6xBVJycR\ncYmkmcDPSF0vfwTeAo6OiGvrHF9zmTULZs6EZZft7EjMzMy6rKpOJVayOvCXiBgMLAUMiohVI+LS\nhkTYTDbaCE47rbOjMDMz69KqnedEwMtk4z8iYkZEvFP3qJrVppt6GnszM7MGqyo5iYj5wEvACo0J\np8mNGAHjx3sqezMzswaqZYbYHwG/yuY16V6GD4cPP4QXXujsSMzMzLqsWs7WuYo0EPbJbEbWhSY8\ni4jl6xFYUxo2DKTUtbPhhp0djZmZWZdUS3IyBqhuWtmuon9/2GADePRROOSQzo7GzMysS6rlVOIr\nGhBH6xg+3INizczMGqjiMSeSekg6VtJDkh6VdLqkTzUyuKY0YgQ8+WSa88TMzMzqrpoBsScAvwA+\nBN4EjgbOK7tFVzRqFEycCH36dHYkZmZmXVI13ToHAUdGxMUAkj4P3CrpW9kpxt3D8l13vK+ZmVkz\nqKblZHXg77k7EXEXaWDsKvUOyszMzLqvapKTXsDHBWVzgN71C8fMzMy6u2q6dQRcISl/JGhf4EJJ\n03MFEfHlegVnZmZm3U81ycmVRcqurlcgZmZmZlBFchIRhzYyEDMzMzOo7do6ZmZmZg3j5KRWN9wA\nu+3W2VGYmZl1OU2TnEg6StIESTMlPSJpeJl195F0p6R3JE2T9LCkXToyXiLgjjvgrbc69LBmZmZd\nXVMkJ5L2A84ETgI2B54E7pA0oMQmnwXuBHYHhgL3ArdI2rQDwk1GjEh/H320ww5pZmbWHTRFckK6\n0vFFEXFVRDwPjAZmAIcVWzkixkTEryNifES8EhEnAC8Be3VYxKuuCoMGOTkxMzOrs4rO1pH0xUp3\nGBE3VxOApN7AMNJ1e3L7CEl3AVtXuA8BSwNTqzn2YpF8hWIzM7MGqPRU4psqXC+AnlXGMCDbZnJB\n+WRg/Qr38T9AP+D6Ko+9eEaMgDPPTONPpA49tJmZWVdVUXISEc3S/bMISQcAJwJfjIgp7a0/ZswY\n+vfvv1DZqFGjGDVqVPUHHz4c3n8fXn4ZBg+ufntrvH/8A7baCpZeetFls2fDEkt0fExmZk1s7Nix\njB07dqGyadOmdWgMiojaN5b6RkTh9Xaq3Udv0viSr+R3CUm6AugfEfuU2XZ/4BLgqxFxezvHGQqM\nHz9+PEOHDl2ckBeYOhVWWAGuvhq+/vX67NPq5/bbYa+94MQT4Sc/WXhZWxvss086JXzYsM6Jz8ys\nRbS1tTEsfVYOi4i2Rh+v6hYRST0lnSjpTeAjSWtn5T+V9M1q9xcRc4DxwE55x1B2/+EycYwCLgX2\nby8xaZjll4ezzoLNNuuUw1sZjzwCX/kK7LorHHfcosvXXRdWWQU+/3kYP77j4zMzs5Jq6a45ATgE\nOBaYnVdb6QB7AAAX/ElEQVT+NPCtGuM4Czhc0kGSNgAuBJYErgCQdJqkT67tk3XlXAn8EHhU0sDs\ntkyNx6/dmDGw0UYdflgr49lnYc89YehQuP566F3kwtnLLJNaVtZfH3beObWkmJlZU6glOTkIOCIi\nrgHm5ZU/CWxQSxARcT3w/4BTgceBTYBdI+LdbJVBwGp5mxxOGkR7HvBW3u03tRzfupDXX4dddoFP\nfxpuuQWWXLL0uv37p4n0Bg9OLSiPP95xcZqZWUnVXJU459PAy0XKewBFfqJWJiLOB84vsezQgvsj\naz2OdWHvvpsSkyWWSEnHssu2v00uQdlll5Sg3H23u+nMzDpZLS0nzwLbFyn/KqnVw6xzHHUUvPce\n3HknrLxy5dstu2zaZu21YaedYOLEhoVoZmbtq6Xl5FTgSkmfJiU3X5a0Pqm75wv1DM6sKueck1pP\n1l23+m1zCcoFF8Dqq9c/NjMzq1jVLScR8VfSNPGfB6aTkpUhwF4R8Y/6hmdWhZVXhk02qX375ZaD\n44+HHk07rY+ZWbdQS8sJEfEAsHOdYzEzMzOr/cJ/kraQdGB2696zWH38MVx4ITz/fGdHYmZm1vJq\nmYRtVUkPAOOAc7Lbo5IelLRqvQNsCb16wQ9/CLfe2tmRmJmZtbxaWk4uIZ0yPCQilo+I5UljTnpk\ny7qfXr3ShF++QnHHmNLuJZTqb/58eOONjj+umVk3VEtysgPwnYh4IVeQ/f894LP1CqzljBjh5KQj\n3H47rLlmmp6+I/3yl+kaPM8+27HHNTPrhmpJTv5D8cnWepJmae2ehg9P82O8+267q1qNctfLGTmy\n4y/Wd/jhMHAgfO5z8NxzHXtsM7Nuppbk5H+A30raIleQ/X8OaQr67mnEiPT30Uc7N46uqpLr5TTS\ngAFp9tgVV0zJkQc/m5k1TEXJiaT3JE2VNBW4HNgM+JekWZJmAf8ChgKXNS7UJrfWWrDCCk5OGqHw\nejmf+lTnxLHiiilBGTAgJSgvvND+NmZmVrVK5zk5pqFRdAVS6trxuJP6quV6OY200kpwzz0pORk5\nEu67D9Zbr3NjMjPrYipKTiLiykYH0iXsuis89VRnR9F1fPwx7LEHvP8+PPhgddfLaaT8BGWvveCZ\nZ9IZW2ZmVheL9YkqqS+wRH5ZRHywWBG1smO6cAPTnDnw0kswZEhqJeoIffrA174GO+9c2/VyGmng\nwJSgTJjgxMTMrM6q/lSV1A/4JbAvsEKRVXoublDWCT78EJZeuvTyxx6DbbZJX8ojR6azVkaOhHXW\naVyyIsGxxzZm3/UwaFC6mZlZXdVyts4ZwOeA7wCzgG8BJ5FOIz6ofqF1QffdlwZ0zprV2ZEkb7wB\nZ50FW24Jm20GEaXX3XTTNObj0EPh1Vdh9GgYPBjWWAMOPhiuvDJNVGZmZraYammP3gs4KCLuk3Q5\n8EBEvCzpNeDrwDV1jbArueYauOQS6N8f9tkH9t8/tUB05GmxkybBn/8M114LDz2Uuk523x3GjEnJ\nRc8SDV9LLpkGpu6yS7o/bRo88ADce2+6PfhgSlLMzMwWUy0tJ8sDr2b/f5DdB3iQxZghVtJRkiZI\nminpEUnD21l/R0njJX0s6UVJzf/N+Pvfw7//Dd/7XkoMdtstDfIcPTp9wc+bV/Guxo4dW92xJ02C\nnXZKp+OOGZMSpCuvhMmT4cYbU6JUKjEppn9/+MIX4Mwzoa0tDQptz7RpJRdVXZ8mN/aSS+AnP4Ff\n/SpdFPKaa+Dmm9Pz/Nhj8OKLzdOC1o4u99y4Pk2rK9UFul59OlItLSevAmsBrwPPk8aejCO1qLxf\nSxCS9gPOBI7I9jUGuEPSehGxyIVUJK0J/A04HzgA+DxwiaS3IuIftcTQYTbeON1OPRWeeAKuuy7d\nLroILr0UDjusot2MHTuWUaNGVX7cFVeE5ZeHiy9OrTbLL9/+NtXo27f88mnT0vwgG264YMzKZz/7\nyanBVdenyY299lpGPf98Gsvz0UfFu7za2mDzzUvv5IYbUuK49NLFb4MGwfbblw/kwQdh+vTSy9dZ\np/xg448+Yuy55zJqvfXSa2a55WCZZaBHzRc073Rd7rXWherTleoCXa8+HamW5ORyYFPgfuB04BZJ\n3yVNaf+DGuMYA1wUEVcBSBoN7AkcRhrjUug7wKsRkRst+YKk7bL9NHdykiOlL6bNN4fTTkvzozRy\nvoyePeFPf2rc/tvTqxdcdlk6w+XGG+Gcc9IX3NChaX6YRx+FmTM7b4K1eltyyQUXCoyAGTNSopJ/\nGzy4/D4++ihNQPfBBwtvN3NmWr7hhu23WB1xRPnp9k86CU4+ufTy115Llw3YYosFZT16pKQyl6xc\nd12ahLCUKVNSK9Fyy6Xnt6PO9jKzllV1chIRZ+f9f5ekDYBhwMsRUfUkH5J6Z9v/Im+/IekuYOsS\nm20F3FVQdgdwdpF1m5+UBqW257//TV8I+R/uM2bArbfCX/+aWl769GlcnIujXz848MB0i0in4N57\nb0pWbrklJU9d9ZRcKdW/X7/qzu456KB0KzR3bkpSZs9ufx93353WL2WZZcpvv956qTvwjDNg6lR4\n7710y/9/qaXK7+OMM1L3FqTJ9HJJTe7vZpvBT39afh+//GWqcyl77JHOJivl9ddTqyGkSw+cemp6\nvfXuveDvIYeUr8uLL8Kbby66Xe5vv36p27ScqVPLDzxfcsnyCfqcOSlZzTd7dvpsyFluufItW9On\npzmESunVK3XblvPqq2kfc+em25w5C/+/3nqw+uqlt3/zzfRjqXDb55+HU05JMXz/++XPIHzyyXQ9\ns/znIf///v3b/8HXiOejcPn8+S3d0thZFvvbICJeA16TtKqkiyPiiCp3MYB0+vHkgvLJwPolthlU\nYv1lJPWJiNbozK/WyJHpF+h++8Fbb8EBB6RxDNOnpxaIN95IzfTNToK11063b34zlX3xix1/vZxW\n1atX+gKqxOJOXNe7d/qAHjq09n18+9uw446LJjW5/9+voDf4hhvS+KhS1l67fHLy7rtw9dXp/8mT\n0zigwi/GL3+5fHJy3nlw7rmll2+zTRpLVs7GG8Pbb5defvbZ5edLGjcOtttu0fIBAxb8/+absMoq\npfdxwgmp5bKUSuqx7bZpHFspv/kNHH106eWvvZbiKEzypkxJSeTcuelim+WSk0sugd/9rvTy7bZL\ng/bL2XDD8q+rc85JSVIpjzySuqfLeeed8j9KTjgBLr+8dJI1bFjq9i/nqKPKJ0kHHrjgZIZinn8e\nfv7z8sf41rfKL6+zev5UXQH4JmncSDPqC/Bcq15RNiI10d95J5x9NtM++oi2adPSGTK77AKrrZbG\ndbS1dXakNZk2bRptLRp7MV2pPnWpS3tzwrS3/wsuaP8Y5fYhpQQHmDZmDG1nF2lknTSp/Bfu3nun\nsVJz56bB6/PmLfh/7tz0C7u9ehx/fPmB0GutVX4fs2bBr3+9UNG0yy6jLX+s2sSJ5eux9dbp86KU\n/v3br8fpp6cWgV69UqtnruUzd7+9ffTtWzRxWOi5eeONBV2jxey/P3zpSws/B/l/l1ii/XqccEL5\nFsg11ii/j9mzF3k+FqrPZZfRNmFC+jFZysorpx9npV5Xffu2X49nnimfnDz55MIJbKEXX2y3i/i5\nBRc7bWeAYX0oyjVpVbMjaVOgLSKqmoQt69aZAXwlIm7OK78C6B8R+xTZ5n5gfET8IK/sEODsiCj6\nk1LSAfg0ZzMzs8Xx9Yj4Y6MP0umd/BExR9J4YCfgZgBJyu6XakP9J7B7QdkuWXkpd5DmYZkIlOlw\nNTMzswJ9gTVJ36UN1+ktJ9m2+wJXAKNZcCrxV4ENIuJdSacBq0TEwdn6awL/Jp1KfBkpkfkNsEdE\nFA6UNTMzsxZSccuJpBvaWaXma9lHxPWSBgCnAgOBJ4BdI+LdbJVBwGp560+UtCfp7JzvA28A33Ri\nYmZm1voqbjnJpqpvV0QculgRmZmZWbdWt24dMzMzs3roFjPDVHvdng6KaXtJN0t6U9J8SV8sss6p\nkt6SNEPSPyStW7C8j6TzJE2R9KGkP0taqWCd5SRdI2mapPckXSKpX53rcpykcZI+kDRZ0o2SFpn9\nqIXqM1rSk9kxpkl6WNJurViXInX7UfZ6O6sV6yPppCz+/NuzrViXvGOtIukPWTwzstfe0IJ1WqJO\nSp+zhc/PfEm/bcG69JD0U0mvZrG+LOnHRdZrifpkx1lK0m8kTczifVDSFgXrNEd9IqJL34D9SGfn\nHARsAFwETAUGdHJcu5HG2OwNzAO+WLD8f7M4vwBsDNwEvAIskbfOBaSzj3YANgceJl0lOn8/fwfa\ngC2AbYAXgavrXJfbgAOBIcBnSNc9mgh8qkXrs2f2/KwDrAv8DJgFDGm1uhQcbzjp2liPA2e16HNz\nEvAUsCKwUnZbvhXrkh1nWWACcAlppuw1SNcKW6sV60Sa72qlvNtOpM+37VuwLscD75A+C1YHvky6\n2O13W/G5yY5zHelkkm2BtbP30/vAys1Wn7pWvBlvwCPAOXn3RRpAe2xnx5YX03wWTU7eAsbk3V8G\nmAnsm3d/FrBP3jrrZ/sakd0fkt3fPG+dXYG5wKAG1mdAdtztukJ9suP8Fzi0VesCLAW8AHwOuJeF\nk5OWqQ/pw7StzPKWqUu239OB+9tZp6XqVBD7b4AXW7EuwC3A7wvK/gxc1aL16QvMAXYrKH8MOLXZ\n6tOlu3W04Lo9d+fKIj1S5a7b0+kkrUU6Qyk/7g+Af7Eg7i1IZ1vlr/MC6WrRuXW2At6LiMfzdn8X\nEEAFF/Op2bLZMaZCa9cna9rdH1gSeLiF63IecEtE3JNf2KL1GazUHfqKpKslrdbCddkLeEzS9Upd\nom2SPpknvEXrlIu9N2luqUtbtC4PAztJGpzFvympxeG2Fq1PL9KlYgqnKJ4JbNds9en0SdgarJbr\n9jSDQaQnsljcuTnABwKzsxdPqXUGkZolPxER8yRNzVunriSJ9GvpwYjIjQVoufpI2pg0qV9f4EPS\nL4UXJG1N69Vlf2Az0gdLoVZ7bh4BDiG1Aq0MnAz8X/Z8tVpdIDWtfwc4E/g5MAI4V9KsiPgDrVmn\nnH2A/sCVeTG0Ul1OJ7UUPC9pHmmM5gkRcW1eHC1Tn4j4SNI/gRMlPZ/FcAApqXiJJqtPV09OrOOd\nD2xI+oXRyp4HNiV9uH4VuEpSO1f4aj6SViUli5+PiDmdHc/iioj82SmfljQOeA3Yl/SctZoewLiI\nODG7/2SWaI0G/tB5YdXFYcDfI6LMhX6a2n6kL+/9gWdJCf45kt7KEsdW9A3SxKVvkrpZ2oA/knoY\nmkqX7tYBppAGYw0sKB8INPMbZhJpbEy5uCcBS0gqvOZ94TqFo6h7AsvTgPpL+h2wB7BjRORferXl\n6hMRcyPi1Yh4PCJOAJ4Ejqb16jKMNHi0TdIcSXNIA9mOljSb9IunleqzkIiYRhpsty6t99wAvA0U\nXo30OdIAzFwsrVYnJK1OGtj7+7ziVqvLGcDpEfGniHgmIq4hTfx5XF4crVQfImJCRIwE+gGrRcRW\nwBKkgfJNVZ8unZxkvxRz1+0BFrpuz8OdFVd7ImIC6UnMj3sZUn9dLu7xpMw3f531SR9quWsM/RNY\nVtLmebvfifQC/Fc9Y84Sk72BkRHxeqvXp4geQJ8WrMtdpDOoNiO1BG1KGgB3NbBpROQ+lFqlPguR\ntBQpMXmrBZ8bgIdYtIt5fVJrUCu/dw4jJb635QpasC5Lkn7c5ptP9r3ZgvX5RETMjIjJkpYjDVa9\nqenqU6+RwM16IzX3zmDhU4n/C6zYyXH1I31RbEZ6wR+T3V8tW35sFudepC+Xm0j9gvmndJ1POg1x\nR9Iv5IdY9JSu20hfRsNJXS0vAH+oc13OB94Dtidl0Llb37x1Wqk+v8jqsgbpdLrTSG/Iz7VaXUrU\nr/BsnZapD/Ar4LPZc7MN8A/Sl+AKrVaX7DhbkAYoHkc6df0A0hin/Vvx+cmOI9Kppj8vsqxl6gJc\nThrouUf2etuHNJbiF61Yn+w4u5CSkTWBnUnTCjwE9Gy2+tS14s16A47M3iwzSVndFk0Q0w6kpGRe\nwe2yvHVOJp3aNYN0Jch1C/bRB/gtqfvqQ+BPwEoF6yxL+pU8jZRA/B5Yss51KVaPecBBBeu1Sn0u\nITVzziT9kriTLDFptbqUqN895CUnrVQfYCxpKoCZpC+OP5I3J0gr1SXvWHuQ5m6ZATwDHFZknZap\nE+lLb15hjK1WF9IPyLNIX8TTSV/SpwC9WrE+2XG+BrycvX/eBM4Blm7G+nj6ejMzM2sqXXrMiZmZ\nmbUeJydmZmbWVJycmJmZWVNxcmJmZmZNxcmJmZmZNRUnJ2ZmZtZUnJyYmZlZU3FyYmZmZk3FyYmZ\nmZk1FScnZi1O0r2Szqpi/TUkzZe0SXZ/h+x+4ZVGG07S5ZJu6Ojj1krSSZIe7+w4zLo6JydmTUbS\nFVmycH6RZedlyy7LK94HOLGKQ7wODAKezitb7OtYVJsktTBf88OswZycmDWfICUQ+0vqkyvM/h8F\nvLbQyhHvR8T0ineevBMR8+sVsC0eSb06OwazZuLkxKw5PQ78B/hyXtmXSYnJQt0KhS0WkiZIOk7S\npZI+kPSapMPzli/UrZNnO0lPSpop6Z+SNsrbZnlJf5T0hqTpkp6StH/e8stJV9o+Otv3PEmrZ8s2\nknSLpGlZPPdLWqugDj+U9JakKZJ+J6lnqQcm17Ui6RtZXd+XNFZSv4LH4PsF2z0u6Sd59+dLOiKL\nbbqkZyVtJWmd7DH9SNJDhbFm2x4h6fVsu+skLV2w/FvZ/mZmf79T5PHfV9J9kmYAB5Sqr1l35OTE\nrDkFcBlwWF7ZYcDlgCrY/gfAo8BmwPnABZIGF+w/n4AzgDHAFsC7wM15SUJf4DFgd2Aj4CLgKklb\nZMuPBv5JujT6QGBl4D+SVgHuJ12ifUdg82yd/JaCzwFrZ8sPAg7JbuWsA+wN7AHsSUqMftTONsX8\nGLgC2BR4DvgjcCHwc2AY6XH5XcE2g0mXnt8T2JVUp0+64CR9nXTZ+eOADYDjgVMlHViwn9OAs4Eh\npEvTm1nGTYlmzesa4HRJq5F+SGwD7AeMrGDbWyPiwuz/X0oak233UlZWLME5OSLuAZB0MPAGaTzL\nnyPiLSB/PMl5knYD9gUei4gPJM0GZkTEu7mVJH0XeB8YFRHzsuJXCo47FfhuRATwoqRbgZ2AS8vU\nT8DBETEjO84fsm2qGXsDcFlE/CXbxxmkBOuUiLgrKzuHlCTm6wMcGBGTsnW+B9wq6YcR8Q4pMflh\nRPw1W/+1rBVqNPCHvP2cnbeOmeVxcmLWpCJiiqS/AYeSvoxvjYipUiUNJ/y74P4kYKVyhwMeyTv2\ne5JeIP2qR1IP4ARSi8GngSWyW3tjXTYFHshLTIp5JktMct4GNm5nvxNziUneNuXqV0r+4zQ5+/t0\nQVlfSUtFxEdZ2eu5xCTzT1LyuL6kj0itOpdKuiRvnZ6kJC3f+BriNesWnJyYNbfLSd0KARxZxXZz\nCu4Hi9eNeyzwPVL3zdOkpOQcUoJSzswK9l1LrO1tM59FW4d6t7OfKFNW6WO3VPb3W8C4gmWFCVrF\ng5jNuhuPOTFrbreTEoBewJ0NPI6ArT65Iy0HrAc8mxVtA/w1IsZGxL+BCdnyfLNJLQT5ngK2LzfA\ntUHeJY17ASCbw2WRga1FVHKa8OqSBuXd35qUeDyfdeu8BawTEa8W3PLPsvLpyGZlODkxa2LZ6b4b\nABsVdH00wk8kfU7SxqRBou8CuTERLwE7S9pa0hDSgNiBBdtPBLbMzkZZISv7HbAMcJ2kYZLWzc6y\nGUxj3QMcKGk7SZ/J6jO3gu2K9ZkVls0CrpS0iaTtSS1I1+WNtTkJOE7S9yQNlrSxpEMkHdPOccws\n4+TErMlFxEd54x2KrtLO/UrWCdLZLueQzvJZEdgrInJf6D8D2kgtOfeQxnjcWLCPX5NaEJ4F3pG0\nekRMJZ2N0w+4j3TGz7dYtFum3k4jnSV0S3a7kUUH4lbyOBUrewm4AbiN9Hg8ARz1ycoRl5LqeCip\n5eg+4GBSa1O545hZRo3/MWZmZmZWObecmJmZWVNxcmJmZmZNxcmJmZmZNRUnJ2ZmZtZUnJyYmZlZ\nU3FyYmZmZk3FyYmZmZk1FScnZmZm1lScnJiZmVlTcXJiZmZmTcXJiZmZmTUVJydmZmbWVP4/44Br\nvCfHaLYAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Compute the moving average loss to smooth out the noise in SGD\n", "plotdata[\"avgloss\"] = moving_average(plotdata[\"loss\"])\n", @@ -551,19 +496,11 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Test data from file data/MNIST/Test-28x28_cntk_text.txt successfully read\n" - ] - } - ], + "outputs": [], "source": [ "# Ensure the training data is read and available for this tutorial\n", "test_file = \"data/MNIST/Test-28x28_cntk_text.txt\"\n", @@ -581,27 +518,19 @@ " labels = StreamDef(field='labels', shape=num_output_classes, is_sparse=False)\n", " )))\n", "\n", - "features_si = mb_source[feature_stream_name]\n", - "labels_si = mb_source[labels_stream_name]\n", + "features_si = test_mb_source[feature_stream_name]\n", + "labels_si = test_mb_source[labels_stream_name]\n", "\n", "print(\"Test data from file {0} successfully read\".format(path))\n" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Average test error: 1.61%\n" - ] - } - ], + "outputs": [], "source": [ "# Test data for trained model\n", "test_minibatch_size = 512\n", @@ -638,7 +567,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": null, "metadata": { "collapsed": false }, @@ -656,7 +585,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": null, "metadata": { "collapsed": false }, @@ -672,7 +601,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": null, "metadata": { "collapsed": false }, @@ -685,20 +614,11 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Label : [7, 9, 0, 1, 7, 9, 0, 0, 9, 2, 9, 3, 8, 5, 7, 8, 8, 0, 8, 4, 6, 8, 6, 2, 3]\n", - "Predicted: [7, 9, 0, 1, 7, 9, 0, 0, 9, 2, 9, 3, 8, 5, 7, 8, 8, 0, 8, 4, 6, 8, 6, 2, 3]\n" - ] - } - ], + "outputs": [], "source": [ "print(\"Label :\", gtlabel[:25])\n", "print(\"Predicted:\", pred[:25])" @@ -713,29 +633,11 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Image Label: 9\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFfCAYAAACfj30KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztnet2IjmzRMXdl2573v8pe7ptc4fz4ztZEwSZkspuW1Q5\n9lpaVZQxBmZ6k6RSqcn5fE5CCCHaMG39BIQQ4jsjCQshREMkYSGEaIgkLIQQDZGEhRCiIZKwEEI0\nRBIWQoiGSMJCCNEQSVgIIRoiCQshREPmrZ/A/6O100KIMTIp3UGRsBBCNEQSFkKIhkjCQgjREElY\nCCEaIgkLIURDJGEhhGiIJCyEEA2RhIUQoiGSsBBCNEQSFkKIhkjCQgjREElYCCEaIgkLIURDJGEh\nhGiIJCyEEA2RhIUQoiGSsBBCNEQSFkKIhkjCQgjREElYCCEaIgkLIURDJGEhhGiIJCyEEA2RhIUQ\noiGSsBBCNEQSFkKIhkjCQgjREElYCCEaIgkLIURDJGEhhGiIJCyEEA2RhIUQoiGSsBBCNEQSFkKI\nhkjCQgjREElYCCEaIgkLIURDJGEhhGiIJCyEEA2RhIUQoiGSsBBCNEQSFkKIhkjCQgjREElYCCEa\nIgkLIURDJGEhhGiIJCyEEA2RhIUQoiGSsBBCNEQSFkKIhkjCQgjREElYCCEaIgkLIURDJGEhhGiI\nJCyEEA2RhIUQoiGSsBBCNEQSFkKIhkjCQgjREElYCCEaIgkLIURDJGEhhGiIJCyEEA2RhIUQoiGS\nsBBCNEQSFkKIhkjCQgjREElYCCEaIgkLIURDJGEhhGjIvPUTEOJ8Pn/4Prmf889q/t57qX3syWRS\n/bPS7b6PJ24LSVjcDOfzuZNY7hidRz+vGUiN9HLPka/Z7+PjeOeTyaQb0+m0eG4Df9c7x+cgbg9J\nWNwEfYRp43Q6FX9u9/HO8TaLqiTOSPR8DX+3Rpiz2SxNp9M0nU678+gay9iTND9vcXtIwqI5nlRZ\nkHyNfx5J9nQ6pePx6B7x3ITJ4opk1ucDwZOlJ2KT63w+d498zYRrgraoF69LwLePJCxugki07xko\n2cPhkI7HYzei25w28CTJEvY+CLyjl0KIri0WizSfz90jnp9Op6soeTqddh8mdq5UxO0jCYubACNH\nlGgUuZpEc9dMsjXjdDplI1YcnO7wBqc5ODr1jibh5XKZlstld87X7G/MZrPuaEJOyU+liNtFEhbN\nwVwqy9cTqzc4wrVr+/0+7ff7i3PvtkWWOfniV/5c9M1pDkwXYNTKt6fTaVqtVmm1WqXlcnlxjq/P\n3i/8e/P55T9l/LAQt40kLG4Cji5zkrVzi2KjaybY3W5XPGIknKtGmE7/V1pfE4nba+FJtWjMZrN0\nd3eXVqtVuru7S3d3d2m/36fVauXmrufzeTgBaH9X3D6SsLgJvHwwyxUla9Fsbux2u4ux3W7d891u\nl47H45V8PRmzhEsDI2GvygHP5/N5uru7S/f3990HBKZLUMDT6fRiMtCuc8pEkfDtIwmL5nhlZRwF\ns3y9I18z2W42m7TdbrPDJOxVHPB5SsmNzL3bnLPlsjO8Np/P08PDw5WAMQWBz4cjYHyOKG1x20jC\n4iaIJMwCrh0W4W42m26YkL1rGAl7qQK8nlK6SoVEEbtJuDRsUq5GwFaiZtc4XYKTgxLx7SMJi0+h\nzz9+FnBJuiaqUp4Xpbter7PnGAmXJs9SShcfDp6QcxLGul8ci8WiEzBPwnEtsVVGcDoCUxzH4zHN\nZrPeIlZVxdciCYtPJVrKi9cwcuVcLd9GAZeGpRo46rXrHHWWBOylI6IjTsx5UvPK3M7nc1osFmm7\n3V6I2v4m/s7xeLyqomCBn06n7u/lVunxNfG1SMLi0/CW8nrnXv4WZYnXvBKzqPzMHtOEy3lgFLqX\njvDqeE1UudI4Lq2L3hes9TUJ73a7TsD49zhvbhK2Cor9fp/u7u4u5I8fdlH5Ha+qk4i/HklYfDql\nZb1R+sBSBXjc7XbZiTpOZXAU7UXVJtLcYgoWVql+GSNcfB/O53OXIrD8LYqYBYwSxnSNSRc/eDCF\n4aUxvOgeqzfs/uJrkYTFpxD1VeDVZCZhSx9sNpv09vaW1ut1ent7uxiehKPh1QnzOd6O6oS92yml\nbE8KPHLqxQaK0Gp6T6dT2m63oYBNshz12+vlD7qULifzuCbZnofdTwJugyQsPg1PxLys1yI7jITX\n63V6fX11B4snt1ijVNKGEXOp0Y4n4dyKOa5Q4PfCi0xxchDfQ64WYQlzCgKXSuNEHuaYLSJPKXWL\nOyThNkjC4lPxaoB5cDrCouDX19f058+f9PLy0g1bWFFTo8vVCl71gl0r9Y5IKV1J2JtY49dmv8cC\ntr8XTQBy/teeJ05YegI28PHm83maz+fuEmd+jeLrkYTFpxDJF6Vh51EkbOL9/ft3+vPnT/rz58+V\nhHN52T4DI+GU/GoCFFUpzeJFpXbuCZ8n/ljCFgEvl8vuveIUhMH1xNb0x5Y5I/j3VVPcBklYfBre\n13BejGGRMOeE397e0svLS/rz50/6/ft3+v37d/r333+7Pg9RHja6VrrNE1l27h2915YbFtl6Use/\nZyLkZdv2TcG6qFlXNZ6Es8dGAZuEOVds99XCjvZIwuJTyQmYv2Z7OWGT8L///pt+/frVSbiUi+UU\nQemIeNLlr+s1pXcsvJLcWcJY6TGfz7vyNYxu+XnjghC733K5zFZMSMBtkYTFp1GKgnG238sJYyT8\n69ev9OvXr7TdbrO5WD5nKUbCjCjlSrn6oc/j8WNbZGrvy2w26wTMu2pgasFbTYcR82q1uvgbPGmn\nSLgtkvA3Br8qG3w7ulbz2N5ElTdphhLGSTnLC6OILRLmXKx3PjRMwrbkmCsb+HZKlxNwHAGbhHe7\n3cXf4IoMFrEm6b4WSfibw3L1ZIsRnjfx5F0r1e+ihE2wv3//Ti8vL11N8Gaz6eqCvT4K5/P5YkLJ\ni0Rry85K4vmImKIo3IvKowUidp5S6j5gvJI1rJ6wsV6v3bw3R8KiDZKwqILLpTiq9SbbWMDeqrbD\n4XAx+fby8pJeX1/Ter2+WFocTZ4Z3td8Lv3Kjdzjebff897VHGsfj8vtWMDL5bJL7ywWi4vXwZN3\nWGUhvh5J+BtTk2bAn+OMvTciyZaOlnKwYRIuRcJRPwQvRxq1j+QmOZHM+VotUUqm72SiN5HIKR1e\npm0C5rpgfN1RnbH4OiThb0z0jy5KSZhAvGW/3tJgrx2ld81ywBYFWzpiu912v8ORcE10601m2eBr\nGA3XlKj1oVSjHNU583V7LPxv4b2fFg3jDs3eBxNHwZJwGyRhcUEuJ4z/8LHNJHcmM3lGPYB5YJ8I\nO/ci4ZSuv07nJq+8LeO924vF4qpfQ7RoA4+172e0pNq75n27wAUc/N8Bfw/TEfY67QPHW8CBj6FI\nuB2SsOiomZTjagbeqcJGLkLmY/QYJvVcJMxRLUa/3vbx0TnvVIHnnpD7vKdeuiY6x+5uu93uQsD2\nQWTnUToCexJzA3msnsgtexZfhyQsXCIhe4srOIK1bmcsFD6PNuLkRu6YE45qYjHqxUg32kKez+0r\ne0pxRQX+rBZcEVgzTJ7b7bb7Oxb5Wg6fJcwC5rQL794R7eAhCbdBEhYppfwCA5wYwpww1vVibe/b\n25u7I0aup28uZ4yRMEoxqotlAeM28ridPF5bLBZ/pZTNe+/6fCBh+sB+32SLJWo8SYoiRgFzftze\nI26BqbxwOyRhcYW3CiwlPyfMDXes6xlHtLnzqAOa12Qnpeu91FAuOEy29/f34dHOWcKlyotaTqeT\nG+V7Uf92u+0EjBHvfr8PxRxJGHPklrbBxRtRG0zx9UjC4oJoGS5HwlYChQLGbmeWF+bJO+/cK82K\nyrS8OlcUsEW/FuU+PDyk+/t794jnJmFPvt61Wk6n09U2TTzsuj0He88xB++lJzAdYb0lWL72fOfz\neff+8N56Ske0RRIWLt7qM2wqg8uMWcT//vuvKxtv2DJkT2yR7LxSK+yTYCkGE+3Dw0N6fHzMHpfL\nZXalGp/XcjqdwolHG8vl8mJRBefeMbXAP+cmP5GEbfEGpn84EhZtkIRHRp9oxos4o8UCx+PxqsG6\n5X+xvAyrI/hrL6cTuLQMJeeJbzqdpsVi0Uk2yvna8CJgGzgxZznkXPTrbXFUg0Wh2PksV/Mcta/0\n9p7zyudK/08o2r09JOFvQq7aIbcE2W7b8mJc1YYTcRYR83JjL9ridIJX8xvV/1q0y6kHb2BOmPPB\nKOCc4LzURB9sAjESMEsYBbxarS5qfqPVfSjg3LXo56ItkvDIyUVBmHcsNdvZ7/edgDkSRhFbJMyP\n40V/Vpvr1flGA3O/npD5monYq4zAKBhXlf3NCgnrimar07zmQihXrMG2HhDRB4U9Bj4WXuNzcZtI\nwiPGy+vyOU7+5MZut7uofjAJcyRsEuaomiVsrRhTSp1ceRVbdA1F6w38mVcbzKkIE7xXD/zREjVL\nR2DPB7vO7SetfzBWnvAHRS4lYY/rnXu3o2via5GER0quvSMvf8VJIG/xhJ1jPpijYRSwlZ1xZQNH\nwimlLs8bRbfekcvRSrdz11DC9twiGePtWiwS9qo7PAmbfL1IHQVsz6X2OSo6vl0k4W9AJORoKTKW\nTeERm+xgFIwTcyZjLnnyvoZbL+Ca1IGdo1A5YvaueX0ieOACidLX/PdEwrkUhKVr7HnYB9l6vQ7z\n1p6Io+f3nucsvhZJeMTkan7tHLcYwlVwJgI8R/FGE3MmkZTiPgw4sTSZTLrVbTiJFg0vRZGTa+3g\n52Tn3vE94Gu3FpKWJ7ZhEjYBexLGD4tcmiSSryLi20MSHjksXb4dNeUxsVpvX1yajNGvV6K23W5T\nSpdb73DdKpadWSSMiyi4vteG9XngfhHcIS3qnZDrLWzUVh3Uvv/2eyhhr6/wfD5P6/XanTzEDwue\nnMPnmIt8Jd3bRBIeIVE5GkvYywl7Ox5zxMttJ71I2NsPDUWGP8d0xP39fXp8fEyPj4/px48fV0fr\neIa1txzVel/fa3bW+Kw8KuaAc3XZi8Uivb29dWV0mLvmSBgfu5RyKL22j74+8TEk4ZHC0vVytKVI\n2HLANrAWmLum4dhsNle7AnMaAielUMIW8f748SP9/Pkz/fz58+KcJZxr2M6CLQ2jJLS+/x1Mvt5/\nC7x9OBzSw8PDxaKTXE44kvFHqjnE1yMJfzNYBJ6EUbK49RDmh3NHa8forZDD29xyEiNhE+/T09PF\nWK1WF8uVvaOd14i1haSiVWuHw+FiYYlXIRF9qPRBYr4tJOERUlqOjEuRsZEO9zTgqNcqJ3glXEr/\n1f5iSqC2npejXswDe0Ji4UY53j6r20rLeT+y3DdXJojXcHFLtO2Qt32TV0PNqwWjlYJe6Zv4WiTh\nkZLbWBKXIqN8o50ycKsh7gXBqQUTRs2yYhsW+eLAr+XR5FQu3fBRvNWFudulx4nSQyhZ7C+cW23o\n7ZIRydfrnYEfjJJweyThEYKRcK4/7+FwcAUcydhrf5jSZX8Ek2Cu7pcHdzWzc5yg4ryoN+nmlZe9\n573j81IDnJyQOfXDC1jw3EoErVyQd73gCgvcKQN7KKOAucoE8838waYcchsk4RGCaQfei4xbIOLi\njNxYr9cXj4fRGeZ77dxKzrh5TnQNIzWO2kwYnGrwOq79zfcwt8jFu3/pvwWnhHhYhQp+2EVd17xU\nBDYusveRP9T4g83eV0XC7ZCERwZHXrwFDgrYGrNjBMxC5lVwHM2llC6WINvPvK/D0Tl2N/NGVCGQ\nq3x473uH72HpPPp9voYpIEwN8TlvfYQfeAZLmJsaYToCo2B8r6N0RN8OceLvIAmPEC8Vwdvg8DLl\nmmjYg1fB2d/3RMCRmfcVudRoJ+o9/LeiYa+Mzysr8+4f3fZSQV6qCNMR0a4X0aQc91HmSJg/5DAd\n4dUfi69DEh4hGAlzlzTcYLJGvDi81W58zc6x5hcXXOAknN32GvPwOaYjcn87pY+XYJUm0XLizUmY\n99DD21gmiJFwLh0R7SqCaR57v7kBvjfZKQm3QRIeKbkdeXmvtygHzMNb5us1YJ/NZlcSsIGLL+ya\nfS3mLeu985rFF3+DPiLOnXMk7PVs5i2jOBLmnaY5EuZ0BEfB9h57rT6VE26PJDxCvOoIjrS4GqIm\nKmYpYi4YpTCfz91I2GqBn56euvOfP39eRLm8EIMHyza6/dH3Ds+9Sgb7Of4en+Pv8X5w3jm2EbXr\nvAty34k5e/+jxkdRdzbxdUjCI4SrI7xt0b1ccE7E6/U6LZfLi4gMUwNWLmX/2DkvGa2Ae3p6cmt/\nOd/rScI7/6iA+TwqL/PuF/2uFwVjbt5LRXiLNuz11U7MYR+OUlc55YTbIQmPDK86wouEMRouRcFY\nHZHSf5GvVyccfS3GdMTz83M3TML2OHz0rn3lexhFwiX54uPw5BtPjmKaKNqfr6ZEzasTtve+tMpQ\nUXA7JOERwhNzXr0wC8BbMosC8Dbp5J4PeOQlyLmeCLjVEYvgb6QWakdUv+sNe/zS8Xw+u9tFRVtK\n2YeiTczZNw98v4/HY0opXZT6cV219x5HC120SKMtkvBIQbl4K+V4cgjl64mGm+5g/pF7AN/f33f5\n3mj1mxd9/U0Bp5SyEuXXyrW8/EHER3xvcnXE0UaqXLNt17A/h/0dFHBK/20JZe8vlvpFO0nnVhh+\n5bcMcY0kPEKiyM6LhnkWHkuichNC+NUXZ+K9cjReJMC7Wdjj/83Xb0eTYFQW5pWOlZZ6o4RzArZr\n3t+JngdGyhbx2offarW6WJHofdvAGuDcApdIxOLrkYRHCou4JGAcXhkWR8K8G8bj4+NF+ZkXHXMn\ntNxX4fdIIVejyykAPvdKxliQ+F7hexyJGL+J1ETY3jeSlP6LfO39Xy6X6XQ6XUiYlyRjJIwpCK6r\nloDbIwmPkCjPWUpJcCRscH0qT8BxE/anpye3R4QJAqMzj75SYPnie4CLVXChCg8vPRClDbB5kVc7\nHE3oRakR/Bm+Hvt9kyZ2qUspXXzTyLX95BK0qOGRZNwGSXik8D/+aAmzlxP2IuGU0pWEvQqIp6en\n9Pz8fJGf9FZqYc0v8lEBc7UCSxhL83CgiL0JNLzOS4mjiT78Gb+v0agp0bNzTAHht43cUu9oxaME\n3A5JeITkImEv2svlhL2JOdySiNMRT09P6Z9//gkbuUeR8N9KPyD44cM7h3D5HbaQ9MrHWMw14vXy\nxLnzlJK7IAbfd6z35ZQPT8xxDXCUipCA2yIJjwwvAuPcYykSxq/GhpVJ5dIRGAnzUmRvN+S+5VGT\nycRNPeDr5vcAqxOwaxxuVMpN6zlNwec5CXvDqz6IaqJNoHbN3ifeqSRqFRpNzOVWGkrGbZGER0iU\ni/RSEjxTH0XB3sQcpyMsEn5+fr5aisz7v+VywrnXVbrO57xQxSJh2yXadpJer9dXC1k4b2zXIgl7\nKQd8/0rR6Gw261Id9l7juSder2G+t0szPg8752uiDZLwiMnlg71IOFeiltJlj4hcOuL5+flqKbKX\n3/zIKq1oMg7P8fVz57j1en2xm7TtoefljfkaSjiX68Xqhmjg+4I7U9t7bL9vdcLYFClq+8k5YXyf\nS8u+xdcjCY8QLyfMIvZKs7zWiUYUCXvpiH/++cf9h/+RJchRKiJ3zYuETcAWBdtO0m9vb1fNjLi/\nBi6k8AQcCZlXq3md50y0JmBcGWcffCZhq0ThXDsuX+bqCEn2dpGER0oUmeUWIniTcl50GU36oeQ/\na5cGnujyJsTsfLPZXES7Ly8vXfrBrts57ibtSRiHSbgkYIuEsVEOvp/4HvEEXK5DmqWB8OelzmiS\n8O0iCY+USJaeOHPLe/lxcHktfq3nSaHPlHBUhcBjs9lcCPjPnz9XQraJOcwJY6WEVz3i/a3oOXo1\n1jxZabdt52krOcPaX959BH+XI2qVng0LSXiERJKqFXAkGayu4Emu19fXC0F81j/+2gj0fD6n7XZ7\nEe1yJIwS9qojvOqRkvi99IjXhJ1TBxbt4pJvLD1DAWOpH5aheSIWt48kPFJyAs7JOJfb5EkuLPfC\nsqjPzEGyaHPH7XbbfUBgNYR3znXC3GGuFAl78jW89pM8oWapBm93ZI6GeWsie9xccx5xu0jCI6c0\nSZdbRstf/b3VZ+v1+mJxADZ6/wz6LAHe7XZX9cCYfsBzq3rgFXLRYhZ+b/E24gnYa8Bu59iQx0tH\nYCSc6w2sSHg4SMIjJIqCP5KSSOm/dARXGvDqrs+MwHIRPV/D51g6Wv0vN+yJ0hH4PnvnSNT8yCpL\nMOLldAT3YfaaIHmtKhUJDwdJeMTk0hEorVoZexNzHAGn9D9RfmY6IteJDF8TNkn3jnhuPSH6VI/Y\n88kdU/JzwrwFEe4Hx93nonSE1yNYkfDwkIRHijdxVZMPzg2OhC0dgf/gTZKfmY5gMUa3eWdpPuI5\npxyic5awvWY8MrkOdNyP2WsB6lVHLJdLtyuaIuHhIQmPmJJ8c+VpUTqC+zBgXwIT8OFw+NR0BNYj\n5yJXe55RDwivH4SXuuHzSMLebVySzM2PcFcSW+yCqQmemOOt6qMl0KoNHhaS8AjxcpU1aYmaiTmM\nhLH/Awp4v99/qoRLu2PwThVRe0o8ry0/iyohctURuBDD25QTW4Hyfn25SDil634Q6hE8PCThkdFH\nvl4uNZeawJww5oBZwNvt9tMlHO2CkTvPTbzZa+b3MXdeS64DHW5LbxKOBkfC+Pj898RwkIRHBkdB\nOGGDX4ltHI/H7ng4HNJ8Pk+Hw8Gd9DFM3Bz1oqg/QwSTycRNReQiYy/9YiK19+d8Pvd6vrmKCO+a\n99+ClyejXHEVHdcC48IMMQ4k4RHC0uXoy3Khx+PRnU3niTgbvEMy3sfEa7nhj0jY+12svOBIHsWH\nk2CTySSdTqcuFZCrKY4i3Cj1UFurLEQJSXiEcF0qSxhXgUUCZglbdIx5Rpaw8d4StdzXav6bnLs2\nuCmOl8utyfHa/aLrpUnB3O8LgUjCI4RXaZmAuR+CRYleBIy51/1+f7VXmd33eDxeRMB47T3PO3c0\nIona/fA5Rs+j9PxyAj6fzxer6bgtqN1PkbCoQRIeGdy1ixcI8EQUyogjYJPKYrHoJuOidERK/6UK\n+tYIe7P5uWv2txHOv9pr88q2vOtMruzMVuNZlch8Pk+73e6iUsS+DSgaFiUk4RES5YSjSoCUrnfh\n4Ch4Pp9fSQujPZPve6LgqKwqV3JVI+toNwtv0rFG8MbxeOwWe8zn84tqEHtPPmuxihgfkvAIwZww\nSpgjYC+ixdVmFunZTL09tuGlA96bhojky9cxovVWhuF1/DbgNbrB65GEvfPj8Xi1iSa+j9bUXpGw\nqEESHiEooMViEZZnmSS44gAXM2CpVE05Fp/XPt8+wyLZ8/ncRbL4wZIrBeMSPRsYuXqvBa9ZKZ8n\nYHv/1LtB1CIJj4woJ+zVx06n0wsBYwrCImDsEYw5Yzx65+95zlG+lq+dTqeLWtkoCubdK3B4u1N4\nHzIGXsNKEbuGqwm1r5vogyQ8Qjgn7AnYfh4J2BrbYPRo98VKCF7+bMc+5MTLt1GW+Hqi12fVIbmx\nWq2yEubjfr+/moTDXhU8gSlEDkl4hGAkGAkYV8xFAsZVXHZfTkGggPDY9/nm8r0Y3bLgMTL2XqNX\nJ+3tahGlW7zjbre7ev1YKcHbzAuRQxIeIZhu4B1+UVLT6fSisU1ucP8F+0pu59GkXE1+mEWLzz9K\nMUSpBrzOWwfljphu4QUr3nXeYDO3u7H3mDVd7XhhSm5hiRgukvCI8SaqWAgYIWIJm/0+TvB5Io4a\n49TkV+08KiXzBk+y5Y6YbsAOZN5ti6i5sZEtPsGfWRtPbIeJTeG9JcteCSBWoFhzef7vNZvN3J7P\nYjxIwiOD86ksX7xfSqmTkLeIg1fd5TqR8YgqKLxzr34Xt+7BY1Td4FU+lHLBOOxbgUX1+/2+e578\nWq1G2JrCe3vReWmbUv7dngc3W8rJXQwfSXiEsIjP57M7oZXSfxKOImATnEXKNVGwSbh28F5p3td8\nFpPXEY6ve5UQ3rn15sXaXl4FiA3icWskjIYPh4PbEMheZy4Kxs078XXYNxBP7mIcSMIjhSVssuPF\nDbaIw5u8wwmt9XpdFG8UDXp5VbxmEa63iIJvo6BKRy9/HJ1j1GqNjfA2pw08AUctMzkSjkSMeWZr\nKYrvpyQ8TiThEYICTsnfbscG/uOOBGzb8JQEjNe4ljh39OSZu11a/Rb1Ts5F0fh8sE8ypiN492ZM\nR3BjJC/1EqUjuBSQl5qziMW4kIRHCqckDJuAwpVmJQHzZpiliNjbqYNn+z0J91ndVkpblCTNt+05\nm2ztmwKXoGEqopQTTum6M50XBS8Wi64PBaZKOMJWJDxOJOER4skXlynjpJ39zBMwRnlcplYTCZeG\niaWUOmApe5N30aReNMHHP7PXZ7W+nI7ASJhzwp4sPRFjlQWvTOQImKNrSXi8SMIjxSI5nGyyYy4C\nLpWhRZEv30YZlc5zy4t5oGxLndG8gR9A/Pu2P549Vkp1OWGvTzOLMqqOwL85m83ScrnsHhPfU/zw\nEuNCEh4huFAAlxhjrwOcFLOv51wPywsJauSL0uDHia55lQxRFUMfueZ6UHir8jA3axJGcZqA1+v1\nVY1wLmLlnLBJfT6fd8ucbWy327RarRQJfyMk4ZHC6YeU/D65tZNn5/M5FK53zvKOpH48HsOaXlxw\ngfW83ko674hVIDXHzWbTRdv4/nmRsN3mdI03gZarE7bctL0uSwNFaQ5JeHxIwiODlw7n+hd4q9py\n1zzRRtc82UaDdxvOLargMjsWrifgGiwiRyHa6+cGPd6Sbi8fHE3OcakaCplTOjbw24sYF5KwuIDz\nx3wNqytyi0A43xpFw1E6Apume30ZWLJ9pdsaL1L3Xs/QXpfojyQssqCAvQUgJmK7rw1b8GDSxjIw\nPtqkIDfi4cjUy+sOSVLRcx3q6xF/B0lYXEW/uSNKmB/jeDxe3MciZvsKbv2LuTrCJMylalwDnJPw\nLcurFOVKhPotAAAQsklEQVQO9XWJv4MkLC7wxIs/MwFjDbL9bDL53wo8O3ri9RZwcGMeXrzhVTxE\nghqasGqlKxmPF0lYXOFVVXA6wvsd+xlGv1xl4Z1jmVyuiQ/+7VI+9VYp5bT5NQ39Q0aUkYRFSuk/\n8Xq3vXQE3g/TD5iCiBr2sJxzK9l4EQZKCv8+XrtVPKF6z39IHyri40jCosMTcUrpIirGXC+mH0ym\nLN/SwD4WpRVwkbDwOASiiDeKhu0cf1+MB0lYXIDi5WsGL3vmFXhcH+vdxmsWSdccS1K6dUF5zzUn\nZb6fGB+S8DeGI1/+mYFpCfy92sUepWPfKoFo4uqWiVIMUQ5Y4v0+SMLfmNrVVznpeUuh+96uSTGM\nSUqlVIRSEN8LSfgb855/2JJBf7wqCK/BUK7pEP6uGBfXtUZCiL8OTjjawBWCuPvzcrm8qpv2tqYS\n40ASFuKTwegWa6KxiTt3j8v1zhDjQhIW4pNBCfP+d9i4CCPhUv8MMR4kYSE+GYyCcYk2ChglzDuJ\n8KpBMS4kYSE+EZxUiyLhXDoCI2GcsBPjQRIW4pPx0hHvmZiTgMeJJCzEJ8PpiI9OzEnE40ISFuKT\n0cScyCEJC/EFeCVqPDGXi4SVjhgvkrAQn4wnYBSxVx3BO4y8dwNTcfto2bIYLX12JvZ6WkS7G/fd\n9Ti3WMPLC0flabgLtBgPkrAYPV47TTu34263uxi4nT2ee1vS8zb3uTagTK45kiLe74EkLEYN7+oR\n7fbhCRiHydgEbEfcsDQSMZLrJ1xqdSnGiSQsRosnXN5k1M5RwFE0XIqEPRFH8ozkKtl+PyRhMWpQ\nwjZYnMfjsSoS5uFJuCYKjvoFl/oqS9DjRBIWowYFjOkDHlFOOBIx/76X8kByAo22ORLfA0lYjBZO\nR5xOpwuB4jnLl9MRnJLw0hAfiYajXTQk4/EjCYtRw5GwyRcn2A6HQ69ImCNgb2LO/naOaJ85/nnu\nXAwfSViMGo6EWcQm11oB27mXW86lI5DS3nk5GYvxIQmL0VIjYBvb7bZYJ4wyjqos7HaJXKladD8x\nTiRhMWo8EbOEvQgYr3mVEVz25tUjMznBej/LLeQQ40ESFqOHI2ETMcq2b3WEt/jDG0xOtqXeEBLw\nOJGExWix6JSlu9ls0na7Tdvttjt/fX1Nr6+v6e3tLa3X6+66CZlL0QzO6Z7P5zSbzbr7zOfzqy5p\n3MDda2PJrSxxt2UxLiRhMVpYwrvdrhPvZrPpZLter9Pr62t6eXlJr6+vroStKgJXwuVyuXZ9Pp93\n0uVxd3d3NXiPOROx+gmPF0lYjBqOhC0CXq/XXdT79vZWjISxNjildCHiaKSU3H7B3jAxewLmfsKS\n8LiQhMVowUgYJ9ws+n17e7sQ8MvLS3fNkzBWP6BscRNO3pATBVszsI9w1M5SjAtJWIyWXDrCJGwp\nCBTwer1O6/U6jISxMQ/2+eV95KbTaTbqjSSMTd+1vdH4kYTFqPEm5lDEr6+v6c+fP10u2EtH4Eo5\nrHrwmrXz8MSbkzH+rrY3+h5IwmK0mDA5J4yRsDchZyMXCRsmYhanHbnyoRQJo8wxolYkPF4kYTFa\nvHSERcIs4ZeXl27SDkvXciVqpW2LovI0PrKES0MSHheSsBgtXp1wFAm/vLxcLNbAZcw4McfNeXAr\ne97Ac7FYXMm2FAnzZB+fS8DjQxIWoyaqjthsNp2ELSfMLSx5tVxtJIxlZrmFGp6E8XHxvFSTLIaL\nJCxGS990hNcjghu5W52w4eWEo63sPQGzjMX3QxIWg8Lbmj66j7eDRkmypT3jDExDoIBZvihaXoih\nRRgiJUlYDBRv23o+Rjtf1I7SdkWcD+ZcsC1NxuoI1f4KRhIWgwNFyztZ4LkXCZcG9whmAZ/P56sV\ncpwLjmqDoyhYq+C+N5KwGCQ1bSTfI99cJNwnHYENejwJYzSsSPh7IwmLweHJkaXp7ar8kXQEk0tH\n5FbJRekI8X2RhMWg4NRAbocLlmqfVERup4ya0jTMCXt9gjUpJwxJWAwST8B83jcdEW3ciX/P6FMd\nwQ3bFQ0LRBIWg8NLRXijT/63787JXjrCm5jjnLAELBhNy4rBUpJxJN6/OTEX5YR5JVyUjtByZKFI\nWAyOGvn2jYpxSXKpRC1qYcnRsLeXXLRlkfi+SMJikPSJgnNRb+2kXLRYoyTgXDpCKQmRktIRYmD0\nkS8uReaoN7c8OaqKyK2a8xq7c1tLFq/kK1KShMWA8WTcp/ohNwmXywcbvLGnt8URS5dL0pQPFpKw\nGBxRJIyy5SiYRRylHCLhMixRbymzdx71B5aIvy/KCYtBkhNxTUrCG/y4JSl/NBKWgEVKkrAYILm8\ncE7ApVrgaGGGXYsoyTgSsFIRIiVJWAwMFiWmFjDlkCtFy03K4WPj3+Pz2iiYlydHMhbfF0lYDI6a\nSJibtJfk64nXE3FK1/ngkoxzeWEhJGExWEq1waV8cFSO5kW/ubwwR7m1AlZeWKQkCYsB8p6FGlE0\njL/Pf8M7N6II2KsbVl5Y5JCExaDI5YSjdESpMiKqgKgpVUspuTLmfHBOwuJ7IwmLweFFwl70Wyvi\n2tpgJoqGWby5dIQ9jvi+aLGGGBwc+eKOyfv9vtvW3ra43+/3ab/fX21d/14Bs3ijfDAuVS6tnBPf\nF0XCYlBw9Ivy3W63abvdps1m0x3t3GTMIu6TcuDbpd4Rnoi5faXSEkISFoOD878W6VoEvN1u03q9\nTuv1uruNETGmJkor4rxr0YIMlq1XK6xIWDCSsBgUUSRsqQeMgD8SCXuRL57n8sDcRa2mVE18XyRh\nMThYwhwJbzabtF6vrwSMIua8MMLC5fNcPjiXjij1kBDfE0lYDIpoUo7TEZGI3yPgKArOCRij4Fwv\nYVVICElYDA4uSzMBR5FwKR0RLcaIjqVmPblImLe5VyQsVKImBoVJE2uBORLO5YS9ibmIkoi9HsIo\n25rJOSH0f4EYFF46AnPCu92ui4QtGsZ64ZqJuVxKoqY+2EtJRKkIRcNCEhaDg9MRVh1hqQeuEfZy\nwl5euHZCLpeKKJWq5Zr4iO+JcsJiUHglahgJm3wxEubVc4fDIZyYQ3jiLBpRVQRHwixiCVikpEhY\nDBCvATvvlly7oSfjpQgi8eaiYD73uqxJxCIlRcJigNRIMZoM6yO+XOQbLdLw/rb2mRM5JGExKLzo\n1BOwd/Rysd7KuJx8c2VpuaMELCIkYTE4vFRBSYpRVQI/Lj6mJ9/JZFIUbi4doRphwUjCYrBwftWb\nIItyt7nHtGNtFJwTsHVNU05YREjCYpDU5oRLtbmeAD35ltIQpQg4SoVIwEISFoOjz6Rc35xsbSWE\nJ9+oOsITOgtYMv6+SMJisNSUjLEkIwF7UqyRvSd+Lx3h5bEVDYuUJGExMEpVC7l+DbXyiyb9SiKO\nInHvMflcfF8kYTFIShNmuYm5Uj4YH7820s6lQLy0g1IRwpCExeCIItUoL8sTYyjGSJC1E3M1Yvae\nuxCGJCwGDS9FjobdF48ennCjnhBeXwi1qxR9kYTFoMA+EbmWlqUt74/H41UPCcwr25F3yJjP52m1\nWqXlcpkWi0VaLBZXe8lxCkKIHJKwGCTcypKbu+Pw+ghzUx9OQXDUi7JdLpdXEo4mAYUoIQmLwcFR\nMEfCKGK8ZveNtjbCNASmH0y2NiwSNhF7UbAELGpRwkoMCszzRhLGBu9eJBxt9MmRMEbDJmCTLwrY\ni4YlYlGLJCwGB/cMzqUjOB/MDd0xGuYJuUjAq9XqIjKO0hFC1CAJi0FRioQ5H8wizm1tVIqETcBe\nThhTEloJJ/ogCYvB4QkYI+H9fu9uacQS9nbYyOWEMRK282gzTwlY1CIJi8FRmpjzomCMhLk6woiq\nIzgXnIuEUcRC1KD/U8Sg4DrhvhNzvN09l6jxIg0vHRFVRygdId6DStTE4Og7MWfXuUTtdDpdPK6J\nk1fI4SRcKRJWOkL0RZGwGCTezsooWS8H7EXAuTphXCnnRcKRgFWiJvogCYtBkuvNy+0ncz2D+TE9\nEXvRcJQPxv7BQtQgCYvBEjVmzw28Pz8Wdk3jdIQ3QVdKR0jEogZJWAyOUvvJqBWlJ2IvGo7K1Gqr\nIiRg0QdJWAwSFnFJvFFKgh+zdtlyjYiFqEESFoMll3aItjTi3+PH8voIR/XCpcoIiVjUIAmLUVCT\nDy6lCWoiYZ6YWywW4X52QtQgCYtB0ke6tTKuqY6oSUdosYbogyQsBoc3mWbHmnRE9DgpXfeOQBHz\nsmXe5ija3FOIHJKwGCx9KiKiibmcwGsjYVzYoeoI0RdJWAwSr9TsPSVq/JicE57NZlc7a3BOWCvm\nxEeQhMWgyFU0eDW+vPNxTshRdUS0zRHnghUJi/egBj5icHiy5RKy1WqVDofDxe9xrwn73cPhcCVo\nlnppSLrivUjCYnDULqhACXvNfliiudt8rTbtIUQJSVgMCo5SvUjYOp0dj8eUki/gw+HgLjX2Fm3k\n0h650jeJWNQgCYvB4VUw8MSZSTgnYMzh5nLHuRSEVxcs+Yo+SMJiUJRWtWFOmLcyOh6PF7W9+/2+\nKhIu5YJrqy+E8JCExeCIROxNzHEEfDgc3BVutdKNImXlg8V7kYTFoMgJ2MZqtUq73a6TsLcjc9R+\n8j0RseQrPoIkLAZHblUbR8ImYKuW2O/3F4srStFwTV7YW5EnIYtaJGExKLylxbl0BEbBuYY7NRUS\nNRNzkq/oiyQsBofJLrcjMqcfLAL2ej1EEW/fyghvBZ4QJSRhMThK6QiOgFHEODAynk6nV4LmDTxr\nF2/YcxSiBklYDAqclEP5YklatI09L+i4u7tLd3d36eHhIU2n0/T8/Jyenp7S8/Nzen5+Tj9//kw/\nfvxIDw8P6f7+Pt3d3VU1dJ9O1ZJF1CMJi0HB1RGLxaKLfG1xBt8P74sCvr+/T/f392m9XqfpdJp+\n/vzZjaenp07Cj4+P6f7+vluJhxF0NMmnSFjUIgmLwcETcsvl8iICrhXwZrNJDw8PabPZpOl0mh4f\nH9OPHz+uhhcJs4S9XTUkYlGDJCwGBacXcFWcwZUTLODtdtuNzWaTttttmkwm6fHxMT08PKSHh4fu\n3I4WCaOEMRWhnTXEe5GExaDgyojFYpHO53M6n89h1YQJeLfbXY3tdpt2u12aTCZdeoKHRc4YCZcE\nLAmLWiRhMTgw0jUB43WulvAqI2zsdru03+/TZDJJd3d3Xd7XzvkaShifhxcFS8SiBklYDAqMdjEH\njOmH4/HYTdiVxn6/T4fDIU0mk6uNPHE/Od7gU0uYxd9CEhaDwoR7Pp/TbDZza4atX0TNsMqKyWTS\nRc/eEc+jlXZ8TYgaJGExOCwSnkwm6XQ6dQK2CTruIZwbNrGHuWQv38u3o4Y/qo4QfZGExaDASBjP\ncaSUrq5Fw6QdRbNepJtboqx8sOjLBFcXNeQmnoQQQvxlip/GWl8phBANkYSFEKIhkrAQQjREEhZC\niIZIwkII0ZBbKVFTPY8Q4luiSFgIIRoiCQshREMkYSGEaIgkLIQQDZGEhRCiIZKwEEI0RBIWQoiG\nSMJCCNEQSVgIIRoiCQshREMkYSGEaIgkLIQQDZGEhRCiIZKwEEI0RBIWQoiGSMJCCNEQSVgIIRoi\nCQshREMkYSGEaIgkLIQQDZGEhRCiIZKwEEI0RBIWQoiGSMJCCNEQSVgIIRoiCQshREMkYSGEaIgk\nLIQQDZGEhRCiIf8H7qwBVftJkgYAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Plot a random image\n", "sample_number = 5\n", @@ -798,7 +700,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.4.5" + "version": "3.4.3" } }, "nbformat": 4, From 78c051ec9007a2e0a985224d39ebe9dc5fa7d771 Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Mon, 2 Jan 2017 19:09:08 -0800 Subject: [PATCH 009/120] CNTK core: Turn on forward compute memory sharing by default --- Source/CNTK/CNTK.cpp | 34 +- .../API/CNTKLibraryInternals.h | 3 + Source/CNTKv2LibraryDll/Common.cpp | 18 +- Source/Common/Globals.cpp | 2 +- Source/Common/Include/Globals.h | 27 +- .../ComputationNetwork.h | 24 +- .../ComputationNetworkEvaluation.cpp | 41 +- Source/ComputationNetworkLib/TrainingNodes.h | 5 + Source/EvalDll/CNTKEval.cpp | 10 +- Source/SGDLib/PostComputingActions.cpp | 1 + Source/SGDLib/SGD.cpp | 11 +- Source/SGDLib/SimpleOutputWriter.h | 10 +- .../Spatial/02_BatchNormConv.ndl | 6 +- .../Spatial/CNTK/baseline.txt | 388 +++++++----------- .../Spatial/CuDNN/baseline.txt | 388 +++++++----------- bindings/csharp/Swig/cntk_cs.i | 2 + 16 files changed, 405 insertions(+), 565 deletions(-) diff --git a/Source/CNTK/CNTK.cpp b/Source/CNTK/CNTK.cpp index 210fa50846f4..912c3015911d 100644 --- a/Source/CNTK/CNTK.cpp +++ b/Source/CNTK/CNTK.cpp @@ -483,6 +483,8 @@ static wstring PathToBSStringLiteral(const wstring& path) // quote a pathname fo return L'"' + path + L'"'; } +// TODO: There is a lot of duplication between this function and the NDL version. +// The code here should be properly refactored to enable sharing. int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wrapper that catches & reports Win32 exceptions { vector args(argv, argv + argc); @@ -581,12 +583,9 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wrapp mpi = MPIWrapper::GetInstance(true /*create*/); } - if (config(L"shareNodeValueMatrices", false)) - Globals::EnableShareNodeValueMatrices(); - if (config(L"hyperCompressMemory", false)) - Globals::EnableHyperCompressMemory(); - if (config(L"optimizeGradientAccumulation", true)) - Globals::EnableGradientAccumulationOptimization(); + Globals::SetShareNodeValueMatrices(config(L"shareNodeValueMatrices", true)); + Globals::SetGradientAccumulationOptimization(config(L"optimizeGradientAccumulation", true)); + Globals::SetHyperCompressMemory(config(L"hyperCompressMemory", false)); TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0)); @@ -728,12 +727,9 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[]) mpi = MPIWrapper::GetInstance(true /*create*/); } - if (config(L"shareNodeValueMatrices", false)) - Globals::EnableShareNodeValueMatrices(); - if (config(L"hyperCompressMemory", false)) - Globals::EnableHyperCompressMemory(); - if (config(L"optimizeGradientAccumulation", true)) - Globals::EnableGradientAccumulationOptimization(); + Globals::SetShareNodeValueMatrices(config(L"shareNodeValueMatrices", true)); + Globals::SetGradientAccumulationOptimization(config(L"optimizeGradientAccumulation", true)); + Globals::SetHyperCompressMemory(config(L"hyperCompressMemory", false)); TracingGPUMemoryAllocator::SetTraceLevel(config(L"traceGPUMemoryAllocations", 0)); @@ -777,12 +773,12 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[]) // This simply merges all the different config parameters specified (eg, via config files or via command line directly), // and prints it. fprintf(stderr, "\nConfiguration, Raw:\n\n"); - LOGPRINTF(stderr, "%s\n", rawConfigString.c_str()); + LOGPRINTF(stderr, "%s\n", rawConfigString.c_str()); // Same as above, but all variables are resolved. If a parameter is set multiple times (eg, set in config, overridden at command line), // All of these assignments will appear, even though only the last assignment matters. fprintf(stderr, "\nConfiguration After Variable Resolution:\n\n"); - LOGPRINTF(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str()); + LOGPRINTF(stderr, "%s\n", config.ResolveVariables(rawConfigString).c_str()); } #endif @@ -793,12 +789,12 @@ int wmainOldCNTKConfig(int argc, wchar_t* argv[]) if (traceLevel > 0) { fprintf(stderr, "\nConfiguration After Processing and Variable Resolution:\n\n"); - config.dumpWithResolvedVariables(); + config.dumpWithResolvedVariables(); - LOGPRINTF(stderr, "Commands:"); - for (int i = 0; i < command.size(); i++) - fprintf(stderr, " %s", command[i].c_str()); - fprintf(stderr, "\n"); + LOGPRINTF(stderr, "Commands:"); + for (int i = 0; i < command.size(); i++) + fprintf(stderr, " %s", command[i].c_str()); + fprintf(stderr, "\n"); } // Setup profiling diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h b/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h index 815c6cc6e680..d1b3643dc5ec 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h @@ -247,7 +247,10 @@ namespace CNTK CNTK_API void SetFixedRandomSeed(unsigned long fixedRandomSeed); CNTK_API void EnableForwardValuesSharing(); + CNTK_API void DisableForwardValuesSharing(); + CNTK_API void EnableHyperMemoryCompress(); + CNTK_API void DisableHyperMemoryCompress(); CNTK_API void EnableGradientAccumulationOptimization(); CNTK_API void DisableGradientAccumulationOptimization(); diff --git a/Source/CNTKv2LibraryDll/Common.cpp b/Source/CNTKv2LibraryDll/Common.cpp index a9190c402ebb..e988cb1a7a81 100644 --- a/Source/CNTKv2LibraryDll/Common.cpp +++ b/Source/CNTKv2LibraryDll/Common.cpp @@ -62,22 +62,32 @@ namespace CNTK void EnableForwardValuesSharing() { - Microsoft::MSR::CNTK::Globals::EnableShareNodeValueMatrices(); + Microsoft::MSR::CNTK::Globals::SetShareNodeValueMatrices(/* enable = */ true); + } + + void DisableForwardValuesSharing() + { + Microsoft::MSR::CNTK::Globals::SetShareNodeValueMatrices(/* enable = */ false); } void EnableHyperMemoryCompress() { - Microsoft::MSR::CNTK::Globals::EnableHyperCompressMemory(); + Microsoft::MSR::CNTK::Globals::SetHyperCompressMemory(/* enable = */ true); + } + + void DisableHyperMemoryCompress() + { + Microsoft::MSR::CNTK::Globals::SetHyperCompressMemory(/* enable = */ false); } void EnableGradientAccumulationOptimization() { - Microsoft::MSR::CNTK::Globals::EnableGradientAccumulationOptimization(); + Microsoft::MSR::CNTK::Globals::SetGradientAccumulationOptimization(/* enable = */ true); } void DisableGradientAccumulationOptimization() { - Microsoft::MSR::CNTK::Globals::DisableGradientAccumulationOptimization(); + Microsoft::MSR::CNTK::Globals::SetGradientAccumulationOptimization(/* enable = */ false); } bool AreEquivalent(const Variable& var1, const Variable& var2, bool allowParameterAndConstantsEquivalence) diff --git a/Source/Common/Globals.cpp b/Source/Common/Globals.cpp index 06d1e81b12bd..81bfd4425f49 100644 --- a/Source/Common/Globals.cpp +++ b/Source/Common/Globals.cpp @@ -13,7 +13,7 @@ namespace Microsoft { namespace MSR { namespace CNTK { std::atomic Globals::m_forceDeterministicAlgorithms(false); std::atomic Globals::m_forceConstantRandomSeed(false); - std::atomic Globals::m_enableShareNodeValueMatrices(false); + std::atomic Globals::m_enableShareNodeValueMatrices(true); std::atomic Globals::m_enableHyperCompressMemory(false); std::atomic Globals::m_optimizeGradientAccumulation(true); diff --git a/Source/Common/Include/Globals.h b/Source/Common/Include/Globals.h index c73f627f39b8..802ad439e816 100644 --- a/Source/Common/Include/Globals.h +++ b/Source/Common/Include/Globals.h @@ -19,32 +19,17 @@ namespace Microsoft { namespace MSR { namespace CNTK { static void ForceConstantRandomSeed() { m_forceConstantRandomSeed = true; } static bool ShouldForceConstantRandomSeed() { return m_forceConstantRandomSeed; } - static void EnableGradientAccumulationOptimization() { m_optimizeGradientAccumulation = true; } - static void DisableGradientAccumulationOptimization() { m_optimizeGradientAccumulation = false; } + static void SetGradientAccumulationOptimization(bool enable) { m_optimizeGradientAccumulation = enable; } static bool ShouldOptimizeGradientAccumulation() { return m_optimizeGradientAccumulation; } // TODO: Currently the flag is set to false. Should be switched to true after more rigorous testing. static bool UseV2Aggregator() { return false; } - static void EnableShareNodeValueMatrices() - { - m_enableShareNodeValueMatrices = true; - } - - static bool ShouldEnableShareNodeValueMatrices() - { - return m_enableShareNodeValueMatrices; - } - - static void EnableHyperCompressMemory() - { - m_enableHyperCompressMemory = true; - } - - static bool ShouldEnableHyperCompressMemory() - { - return m_enableHyperCompressMemory; - } + static void SetShareNodeValueMatrices(bool enable) { m_enableShareNodeValueMatrices = enable; } + static bool ShouldEnableShareNodeValueMatrices() { return m_enableShareNodeValueMatrices; } + + static void SetHyperCompressMemory(bool enable) { m_enableHyperCompressMemory = enable; } + static bool ShouldEnableHyperCompressMemory() { return m_enableHyperCompressMemory; } private: static std::atomic m_forceDeterministicAlgorithms; diff --git a/Source/ComputationNetworkLib/ComputationNetwork.h b/Source/ComputationNetworkLib/ComputationNetwork.h index 6420e29f05f2..ba9792815bd8 100644 --- a/Source/ComputationNetworkLib/ComputationNetwork.h +++ b/Source/ComputationNetworkLib/ComputationNetwork.h @@ -139,7 +139,8 @@ class ComputationNetwork : template // version that takes multiple nodes void ForwardProp(const NODESET& nodes) { - for (auto& node : nodes) + auto nodesSortedByGlobalEvalOrder = SortByGlobalEvalOrder(nodes); + for (auto& node : nodesSortedByGlobalEvalOrder) ForwardProp(node); } @@ -196,7 +197,7 @@ class ComputationNetwork : private: void PrintMemorySharingStructure(const std::vector& nodes); - void ReleaseMatricesAfterEvalForChildren(ComputationNodeBasePtr n, std::unordered_map& parentCount); + void ReleaseMatricesAfterEvalForChildren(ComputationNodeBasePtr n, std::unordered_map>& parentsMap); void AllocateGradientMatricesForInputs(ComputationNodeBasePtr parentNode); public: @@ -255,6 +256,25 @@ class ComputationNetwork : m_evalOrders[rootNode] = evalOrder; } + template + std::vector SortByGlobalEvalOrder(const ContainerType& nodesToSort) + { + std::vector sortedEvalOrder; + if (nodesToSort.size() == 1) + sortedEvalOrder.assign(nodesToSort.cbegin(), nodesToSort.cend()); + else + { + const std::list& allNodesEvalOrder = GetEvalOrder(nullptr); + for (auto& node : allNodesEvalOrder) + { + if (std::find(nodesToSort.cbegin(), nodesToSort.cend(), node) != nodesToSort.cend()) + sortedEvalOrder.push_back(node); + } + } + + return sortedEvalOrder; + } + // replace an existing eval order with an updated one // This is meant to be used by FormRecurrentLoops(). TODO: Hopefully this can be not done anymore some day. void UpdateEvalOrder(const ComputationNodeBasePtr& rootNode, std::list& nodes) diff --git a/Source/ComputationNetworkLib/ComputationNetworkEvaluation.cpp b/Source/ComputationNetworkLib/ComputationNetworkEvaluation.cpp index 1c5f17f0e4e9..f28591a73688 100644 --- a/Source/ComputationNetworkLib/ComputationNetworkEvaluation.cpp +++ b/Source/ComputationNetworkLib/ComputationNetworkEvaluation.cpp @@ -985,7 +985,7 @@ void ComputationNetwork::AllocateAllMatrices(const std::vector 0) - fprintf(stderr, "\n\nAllocating matrices for forward and/or backward propagation.\n"); + fprintf(stderr, "\n\nAllocating matrices for forward and/or backward propagation.\n"); VerifyIsCompiled("AllocateAllMatrices"); @@ -1026,18 +1026,13 @@ void ComputationNetwork::AllocateAllMatrices(const std::vectorNeedsGradient() && node->InputUsedInComputingInputNodesGradients(i)); } else - { outputValueNeededDuringBackProp[input] = false; - } } } } - std::unordered_map parentCount; for (auto& keyValue : parentsMap) { - parentCount[keyValue.first] = keyValue.second.size(); - // Indicate on the node that it's parent overwrites its gradient if the node is not part of a loop // and has exactly one parent who implements the gradient overwrite optimization if (Globals::ShouldOptimizeGradientAccumulation() && @@ -1057,16 +1052,9 @@ void ComputationNetwork::AllocateAllMatrices(const std::vector& allNodesEvalOrder = GetEvalOrder(nullptr); + std::list nodesForForwardPropRoots = ComputationNodeBase::EnumerateNodes(forwardPropRoots); - std::vector compositeForwardPropEvalOrder; - for (auto& node : allNodesEvalOrder) - { - if (std::find(nodesForForwardPropRoots.cbegin(), nodesForForwardPropRoots.cend(), node) != nodesForForwardPropRoots.cend()) - { - compositeForwardPropEvalOrder.push_back(node); - } - } + std::vector compositeForwardPropEvalOrder = SortByGlobalEvalOrder(nodesForForwardPropRoots); set completedEvaluate; for (auto& nodeIter : compositeForwardPropEvalOrder) @@ -1083,17 +1071,15 @@ void ComputationNetwork::AllocateAllMatrices(const std::vectorRequestMatricesBeforeForwardProp(m_matrixPool); for (auto& nodeLoopIter : recInfo->m_nestedNodes) - { - ReleaseMatricesAfterEvalForChildren(nodeLoopIter, parentCount); - } + ReleaseMatricesAfterEvalForChildren(nodeLoopIter, parentsMap); } } else { nodeIter->RequestMatricesBeforeForwardProp(m_matrixPool); - // we only release matrices for the children since the root node's information will be used and should not be shared - // with others - ReleaseMatricesAfterEvalForChildren(nodeIter, parentCount); + // we only release matrices for the children since the root node's information will be used + // and should not be shared with others + ReleaseMatricesAfterEvalForChildren(nodeIter, parentsMap); } } @@ -1139,17 +1125,20 @@ void ComputationNetwork::AllocateAllMatrices(const std::vector 0) - PrintMemorySharingStructure(GetAllNodes()); + PrintMemorySharingStructure(GetAllNodes()); } -void ComputationNetwork::ReleaseMatricesAfterEvalForChildren(ComputationNodeBasePtr n, std::unordered_map& parentCount) +void ComputationNetwork::ReleaseMatricesAfterEvalForChildren(ComputationNodeBasePtr n, std::unordered_map>& parentsMap) { for (int i = 0; i < n->GetNumInputs(); i++) { ComputationNodeBasePtr pNode = n->GetInputs()[i]; - parentCount[pNode]--; - if (parentCount[pNode] == 0) - pNode->ReleaseMatricesAfterForwardProp(m_matrixPool); + if (!parentsMap[pNode].empty()) + { + parentsMap[pNode].erase(n); + if (parentsMap[pNode].empty()) + pNode->ReleaseMatricesAfterForwardProp(m_matrixPool); + } } } diff --git a/Source/ComputationNetworkLib/TrainingNodes.h b/Source/ComputationNetworkLib/TrainingNodes.h index 77b1bc96996b..49b0321354b8 100644 --- a/Source/ComputationNetworkLib/TrainingNodes.h +++ b/Source/ComputationNetworkLib/TrainingNodes.h @@ -2586,6 +2586,11 @@ class BatchNormalizationNode : public ComputationNodeNonLooping, publi if (isFinalValidationPass) { + // The current implementation requires that the gradient of the first operand/input be computed + // in order to compute gradients for the bias and scale parameters (2nd and 3rd inputs) + if ((Input(1)->NeedsGradient() || Input(2)->NeedsGradient()) && !Input(0)->NeedsGradient()) + InvalidArgument("%ls %ls currently supports learnable scale and bias parameters only if the first input also needs gradient (i.e. is dependent on at-least one learnable parameter).", NodeName().c_str(), OperationName().c_str()); + if (m_convertRunningVariancePending) { // Prior to CNTK CuDNN v5 support (and the CNTK engine of the same time), mean and inverse standard deviation diff --git a/Source/EvalDll/CNTKEval.cpp b/Source/EvalDll/CNTKEval.cpp index 51b7562b5997..02de230fa2e7 100644 --- a/Source/EvalDll/CNTKEval.cpp +++ b/Source/EvalDll/CNTKEval.cpp @@ -40,10 +40,9 @@ void CNTKEvalBase::Init(const std::string& config) m_config.Parse(config); size_t nThreads = m_config("numCPUThreads", "1"); CPUMatrix::SetNumThreads(nThreads); - if (m_config(L"shareNodeValueMatrices", false)) - Globals::EnableShareNodeValueMatrices(); - if (m_config(L"hyperCompressMemory", false)) - Globals::EnableHyperCompressMemory(); + + Globals::SetShareNodeValueMatrices(m_config(L"shareNodeValueMatrices", true)); + Globals::SetHyperCompressMemory(m_config(L"hyperCompressMemory", false)); } @@ -377,11 +376,12 @@ void CNTKEvalExtended::ForwardPassT(const std::vectorm_net->ForwardProp(m_outputNodes); for (size_t i2 = 0; i2 < m_outputNodes.size(); ++i2) { auto node = m_outputNodes[i2]; - this->m_net->ForwardProp(node); + shared_ptr> outputMatrix = dynamic_pointer_cast>(node->ValuePtr()); auto pMBLayout = node->GetMBLayout(); if (!pMBLayout) diff --git a/Source/SGDLib/PostComputingActions.cpp b/Source/SGDLib/PostComputingActions.cpp index c0efee5f803c..d67a7e3b3efd 100644 --- a/Source/SGDLib/PostComputingActions.cpp +++ b/Source/SGDLib/PostComputingActions.cpp @@ -75,6 +75,7 @@ void PostComputingActions::BatchNormalizationStatistics(IDataReader * else dataReader->StartMinibatchLoop(mbSize, 0, inputMatrices.GetStreamDescriptions(), totalEpochSize); + bnNodes = m_net->SortByGlobalEvalOrder(bnNodes); for (auto& node : bnNodes) { let bnNode = static_pointer_cast>(node); diff --git a/Source/SGDLib/SGD.cpp b/Source/SGDLib/SGD.cpp index 1d6ca807e83c..1a86b22b70ed 100644 --- a/Source/SGDLib/SGD.cpp +++ b/Source/SGDLib/SGD.cpp @@ -1014,6 +1014,9 @@ size_t SGD::TrainOneEpoch(ComputationNetworkPtr net, epochStartSample = trainSetDataReader->GetCurrentSamplePosition(); } + auto forwardPropRoots = evaluationNodes; + forwardPropRoots.push_back(criterionNodes[0]); + bool noMoreSamplesToProcess = false; bool isFirstMinibatch = true; for (;;) @@ -1104,13 +1107,7 @@ size_t SGD::TrainOneEpoch(ComputationNetworkPtr net, // compute eval node first since when gradient is computed the forward function values // may be changed and need to be recomputed when gradient and function value share the same matrix - net->ForwardProp(evaluationNodes); // the bulk of this evaluation is reused in ComputeGradient() below - - // =========================================================== - // forward prop for training criterion - // =========================================================== - - net->ForwardProp(criterionNodes[0]); + net->ForwardProp(forwardPropRoots); // the bulk of this evaluation is reused in ComputeGradient() below // =========================================================== // backprop diff --git a/Source/SGDLib/SimpleOutputWriter.h b/Source/SGDLib/SimpleOutputWriter.h index 7accba1cbe8d..620708877e42 100644 --- a/Source/SGDLib/SimpleOutputWriter.h +++ b/Source/SGDLib/SimpleOutputWriter.h @@ -65,12 +65,10 @@ class SimpleOutputWriter while (DataReaderHelpers::GetMinibatchIntoNetwork(dataReader, m_net, nullptr, false, false, inputMatrices, actualMBSize, nullptr)) { ComputationNetwork::BumpEvalTimeStamp(inputNodes); + m_net->ForwardProp(outputNodes); for (int i = 0; i < outputNodes.size(); i++) - { - m_net->ForwardProp(outputNodes[i]); outputMatrices[outputNodes[i]->NodeName()] = (void*) (&dynamic_pointer_cast>(outputNodes[i])->Value()); - } if (doWriterUnitTest) { @@ -109,11 +107,9 @@ class SimpleOutputWriter std::map outputMatrices; + m_net->ForwardProp(outputNodes); for (int i = 0; i < outputNodes.size(); i++) - { - m_net->ForwardProp(outputNodes[i]); outputMatrices[outputNodes[i]->NodeName()] = (void*)(&dynamic_pointer_cast>(outputNodes[i])->Value()); - } // TODO: What should the data size be? dataWriter.SaveData(0, outputMatrices, 1, 1, 0); @@ -237,12 +233,12 @@ class SimpleOutputWriter for (size_t numMBsRun = 0; DataReaderHelpers::GetMinibatchIntoNetwork(dataReader, m_net, nullptr, false, false, inputMatrices, actualMBSize, nullptr); numMBsRun++) { ComputationNetwork::BumpEvalTimeStamp(inputNodes); + m_net->ForwardProp(outputNodes); for (auto & onode : outputNodes) { // compute the node value // Note: Intermediate values are memoized, so in case of multiple output nodes, we only compute what has not been computed already. - m_net->ForwardProp(onode); FILE* file = *outputStreams[onode]; WriteMinibatch(file, dynamic_pointer_cast>(onode), formattingOptions, formatChar, valueFormatString, labelMapping, numMBsRun, /* gradient */ false); diff --git a/Tests/EndToEndTests/BatchNormalization/Spatial/02_BatchNormConv.ndl b/Tests/EndToEndTests/BatchNormalization/Spatial/02_BatchNormConv.ndl index eb04ba04f492..627b281d6f3d 100644 --- a/Tests/EndToEndTests/BatchNormalization/Spatial/02_BatchNormConv.ndl +++ b/Tests/EndToEndTests/BatchNormalization/Spatial/02_BatchNormConv.ndl @@ -37,13 +37,17 @@ DNN=[ cMap1 = 3 hStride1 = 1 vStride1 = 1 + # weight[cMap1, kW1 * kH1 * ImageC] + W = LearnableParameter(cMap1, 75, init = Gaussian, initValueScale = conv1WScale, initOnCPUOnly=true) + c = Convolution(W, featScaled, kW1, kH1, cMap1, hStride1, vStride1, zeroPadding = true, imageLayout = $imageLayout$) + b = LearnableParameter(cMap1, 1, init = fixedValue, value = conv1BValue) sc = LearnableParameter(cMap1, 1, init = fixedValue, value = scValue) m = LearnableParameter(cMap1, 1, init = fixedValue, value = 0, learningRateMultiplier = 0) v = LearnableParameter(cMap1, 1, init = fixedValue, value = 0, learningRateMultiplier = 0) - y = BatchNormalization(featScaled, sc, b, m, v, spatial = true, normalizationTimeConstant = bnTimeConst, imageLayout = $imageLayout$, engine=$batchNormalizationEngine$) + y = BatchNormalization(c, sc, b, m, v, spatial = true, normalizationTimeConstant = bnTimeConst, imageLayout = $imageLayout$, engine=$batchNormalizationEngine$) conv1 = RectifiedLinear(y) # pool1 diff --git a/Tests/EndToEndTests/BatchNormalization/Spatial/CNTK/baseline.txt b/Tests/EndToEndTests/BatchNormalization/Spatial/CNTK/baseline.txt index 990d9f0f4acb..f5f5576f8642 100644 --- a/Tests/EndToEndTests/BatchNormalization/Spatial/CNTK/baseline.txt +++ b/Tests/EndToEndTests/BatchNormalization/Spatial/CNTK/baseline.txt @@ -1,60 +1,44 @@ CPU info: - CPU Model Name: Intel(R) Xeon(R) CPU E5-2630 v2 @ 2.60GHz + CPU Model Name: Intel(R) Xeon(R) CPU E5-2620 0 @ 2.00GHz Hardware threads: 24 - Total Memory: 268381192 kB + Total Memory: 33476276 kB ------------------------------------------------------------------- -=== Running /cygdrive/c/jenkins/workspace/CNTK-Test-Windows-W1/x64/release/cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.cntk currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn -------------------------------------------------------------------- -Build info: - - Built time: Aug 23 2016 03:29:14 - Last modified date: Mon Aug 22 21:29:13 2016 - Build type: Release - Build target: GPU - With 1bit-SGD: no - Math lib: mkl - CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5 - CUB_PATH: C:\src\cub-1.4.1 - CUDNN_PATH: C:\NVIDIA\cudnn-5.0\cuda - Build Branch: HEAD - Build SHA1: 66498cf414f4bbf6392730c2abfbef6a5eeb8f7b - Built by svcphil on LIANA-09-w - Build Path: c:\jenkins\workspace\CNTK-Build-Windows\Source\CNTK\ -------------------------------------------------------------------- -Changed current directory to C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -08/23/2016 03:52:13: ------------------------------------------------------------------- -08/23/2016 03:52:13: Build info: - -08/23/2016 03:52:13: Built time: Aug 23 2016 03:29:14 -08/23/2016 03:52:13: Last modified date: Mon Aug 22 21:29:13 2016 -08/23/2016 03:52:13: Build type: Release -08/23/2016 03:52:13: Build target: GPU -08/23/2016 03:52:13: With 1bit-SGD: no -08/23/2016 03:52:13: Math lib: mkl -08/23/2016 03:52:13: CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5 -08/23/2016 03:52:13: CUB_PATH: C:\src\cub-1.4.1 -08/23/2016 03:52:13: CUDNN_PATH: C:\NVIDIA\cudnn-5.0\cuda -08/23/2016 03:52:13: Build Branch: HEAD -08/23/2016 03:52:13: Build SHA1: 66498cf414f4bbf6392730c2abfbef6a5eeb8f7b -08/23/2016 03:52:13: Built by svcphil on LIANA-09-w -08/23/2016 03:52:13: Build Path: c:\jenkins\workspace\CNTK-Build-Windows\Source\CNTK\ -08/23/2016 03:52:13: ------------------------------------------------------------------- -08/23/2016 03:52:14: ------------------------------------------------------------------- -08/23/2016 03:52:14: GPU info: - -08/23/2016 03:52:14: Device[0]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/23/2016 03:52:14: Device[1]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/23/2016 03:52:14: Device[2]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/23/2016 03:52:14: ------------------------------------------------------------------- - -08/23/2016 03:52:14: Running on DPHAIM-25 at 2016/08/23 03:52:14 -08/23/2016 03:52:14: Command line: -C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\release\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.cntk currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn - - - -08/23/2016 03:52:14: >>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>> -08/23/2016 03:52:14: RootDir = "." +=== Running /cygdrive/e/NetScale/CNTK/git_repos/git_cntkv2Library_5/x64/debug/cntk.exe configFile=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.cntk currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn +CNTK 2.0.beta7.0+ (amitaga/dataInteropFeatures 5ea312, Jan 3 2017 23:37:25) on Amitaga-Win-DT3 at 2017/01/04 07:42:50 + +E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\x64\debug\cntk.exe configFile=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.cntk currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn +Changed current directory to C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +01/04/2017 07:42:52: ------------------------------------------------------------------- +01/04/2017 07:42:52: Build info: + +01/04/2017 07:42:52: Built time: Jan 3 2017 23:37:25 +01/04/2017 07:42:52: Last modified date: Tue Jan 3 18:50:06 2017 +01/04/2017 07:42:52: Build type: Debug +01/04/2017 07:42:52: Build target: GPU +01/04/2017 07:42:52: With 1bit-SGD: yes +01/04/2017 07:42:52: With ASGD: no +01/04/2017 07:42:52: Math lib: mkl +01/04/2017 07:42:52: CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.0 +01/04/2017 07:42:52: CUB_PATH: C:\cub-1.4.1 +01/04/2017 07:42:52: CUDNN_PATH: C:\cudnn-5.1 +01/04/2017 07:42:52: Build Branch: amitaga/dataInteropFeatures +01/04/2017 07:42:52: Build SHA1: 5ea3126709951e8b903eeffbd2c3bc1b6bf1db53 (modified) +01/04/2017 07:42:52: Built by amitaga on Amitaga-Win-DT3 +01/04/2017 07:42:52: Build Path: E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Source\CNTK\ +01/04/2017 07:42:52: MPI distribution: msmpi +01/04/2017 07:42:52: MPI version: 7.0.12437.6 +01/04/2017 07:42:52: ------------------------------------------------------------------- +01/04/2017 07:42:53: ------------------------------------------------------------------- +01/04/2017 07:42:53: GPU info: + +01/04/2017 07:42:53: Device[0]: cores = 1536; computeCapability = 3.0; type = "GeForce GTX 690"; memory = 2048 MB +01/04/2017 07:42:53: Device[1]: cores = 384; computeCapability = 3.0; type = "GeForce GTX 650"; memory = 1024 MB +01/04/2017 07:42:53: Device[2]: cores = 1536; computeCapability = 3.0; type = "GeForce GTX 690"; memory = 2048 MB +01/04/2017 07:42:53: ------------------------------------------------------------------- + +Configuration, Raw: + +01/04/2017 07:42:53: RootDir = "." ConfigDir = "$RootDir$" DataDir = "$RootDir$" OutputDir = "$RootDir$/Output" @@ -117,24 +101,24 @@ Test = [ ] ] ] -currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu -DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial -OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu +currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu +DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial +OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn -08/23/2016 03:52:14: <<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<< -08/23/2016 03:52:14: >>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>> -08/23/2016 03:52:14: RootDir = "." +Configuration After Variable Resolution: + +01/04/2017 07:42:53: RootDir = "." ConfigDir = "." DataDir = "." OutputDir = "./Output" -ModelDir = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models" -ndlMacros = "C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/Macros.ndl" +ModelDir = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models" +ndlMacros = "E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/Macros.ndl" precision = "float" deviceId = 0 imageLayout = "cudnn" @@ -145,9 +129,9 @@ traceLevel = 1 numMBsToShowResult = 500 Train = [ action = "train" - modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv" + modelPath = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv" NDLNetworkBuilder = [ - networkDescription = "C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.ndl" + networkDescription = "E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.ndl" ] SGD = [ epochSize = 1024 @@ -160,7 +144,7 @@ Train = [ ] reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData/Train_cntk_text.txt" + file = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData/Train_cntk_text.txt" input = [ features = [ dim = 3072 @@ -175,11 +159,11 @@ Train = [ ] Test = [ action = "test" - modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv" + modelPath = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv" minibatchSize = 16 reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData/Test_cntk_text.txt" + file = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData/Test_cntk_text.txt" input = [ features = [ dim = 3072 @@ -192,40 +176,40 @@ Test = [ ] ] ] -currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu -DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial -OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu +currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu +DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial +OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn -08/23/2016 03:52:14: <<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<< -08/23/2016 03:52:14: >>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>> +Configuration After Processing and Variable Resolution: + configparameters: 02_BatchNormConv.cntk:batchNormalizationEngine=cudnn configparameters: 02_BatchNormConv.cntk:command=Train:Test -configparameters: 02_BatchNormConv.cntk:ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial -configparameters: 02_BatchNormConv.cntk:currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -configparameters: 02_BatchNormConv.cntk:DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData +configparameters: 02_BatchNormConv.cntk:ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial +configparameters: 02_BatchNormConv.cntk:currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +configparameters: 02_BatchNormConv.cntk:DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData configparameters: 02_BatchNormConv.cntk:deviceId=0 configparameters: 02_BatchNormConv.cntk:imageLayout=cudnn configparameters: 02_BatchNormConv.cntk:initOnCPUOnly=true -configparameters: 02_BatchNormConv.cntk:ModelDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models -configparameters: 02_BatchNormConv.cntk:ndlMacros=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/Macros.ndl +configparameters: 02_BatchNormConv.cntk:ModelDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models +configparameters: 02_BatchNormConv.cntk:ndlMacros=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/Macros.ndl configparameters: 02_BatchNormConv.cntk:numMBsToShowResult=500 -configparameters: 02_BatchNormConv.cntk:OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu +configparameters: 02_BatchNormConv.cntk:OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu configparameters: 02_BatchNormConv.cntk:precision=float configparameters: 02_BatchNormConv.cntk:RootDir=. -configparameters: 02_BatchNormConv.cntk:RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu +configparameters: 02_BatchNormConv.cntk:RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu configparameters: 02_BatchNormConv.cntk:Test=[ action = "test" - modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv" + modelPath = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv" minibatchSize = 16 reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData/Test_cntk_text.txt" + file = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData/Test_cntk_text.txt" input = [ features = [ dim = 3072 @@ -243,9 +227,9 @@ configparameters: 02_BatchNormConv.cntk:timestamping=true configparameters: 02_BatchNormConv.cntk:traceLevel=1 configparameters: 02_BatchNormConv.cntk:Train=[ action = "train" - modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv" + modelPath = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv" NDLNetworkBuilder = [ - networkDescription = "C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.ndl" + networkDescription = "E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.ndl" ] SGD = [ epochSize = 1024 @@ -258,7 +242,7 @@ configparameters: 02_BatchNormConv.cntk:Train=[ ] reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData/Train_cntk_text.txt" + file = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData/Train_cntk_text.txt" input = [ features = [ dim = 3072 @@ -272,185 +256,110 @@ configparameters: 02_BatchNormConv.cntk:Train=[ ] ] -08/23/2016 03:52:14: <<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<< -08/23/2016 03:52:14: Commands: Train Test -08/23/2016 03:52:14: Precision = "float" -08/23/2016 03:52:14: CNTKModelPath: C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv -08/23/2016 03:52:14: CNTKCommandTrainInfo: Train : 2 -08/23/2016 03:52:14: CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : 2 - -08/23/2016 03:52:14: ############################################################################## -08/23/2016 03:52:14: # # -08/23/2016 03:52:14: # Action "train" # -08/23/2016 03:52:14: # # -08/23/2016 03:52:14: ############################################################################## +01/04/2017 07:42:53: Commands: Train Test +01/04/2017 07:42:53: precision = "float" -08/23/2016 03:52:14: CNTKCommandTrainBegin: Train +01/04/2017 07:42:53: ############################################################################## +01/04/2017 07:42:53: # # +01/04/2017 07:42:53: # Train command (train action) # +01/04/2017 07:42:53: # # +01/04/2017 07:42:53: ############################################################################## -08/23/2016 03:52:14: Creating virgin network. +01/04/2017 07:42:53: +Creating virgin network. NDLBuilder Using GPU 0 -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.000000. -Node 'b' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'sc' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'm' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'var' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'h1.W' (LearnableParameter operation): Initializing Parameter[64 x 15 x 15 x 3] <- 0.000000. -Node 'h1.b' (LearnableParameter operation): Initializing Parameter[64 x 1] <- 0.000000. -Node 'OutputNodes.W' (LearnableParameter operation): Initializing Parameter[10 x 64] <- 0.000000. -Node 'OutputNodes.b' (LearnableParameter operation): Initializing Parameter[10] <- 0.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'b' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'sc' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 1.000000. -Node 'm' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'var' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'h1.W' (LearnableParameter operation): Initializing Parameter[64 x 15 x 15 x 3] <- gaussian(seed=1, range=0.007698*12.000000, onCPU=false). -Microsoft::MSR::CNTK::GPUMatrix::SetGaussianRandomValue (GPU): creating curand object with seed 1, sizeof(ElemType)==4 -Node 'h1.b' (LearnableParameter operation): Initializing Parameter[64 x 1] <- 0.000000. -Node 'OutputNodes.W' (LearnableParameter operation): Initializing Parameter[10 x 64] <- gaussian(seed=2, range=0.025000*1.500000, onCPU=false). -Node 'OutputNodes.b' (LearnableParameter operation): Initializing Parameter[10] <- 0.000000. - -Post-processing network... - -3 roots: - CE = CrossEntropyWithSoftmax() - Err = ErrorPrediction() - OutputNodes.z = Plus() - -Validating network. 22 nodes to process in pass 1. - -Validating --> labels = InputValue() : -> [10 x *] -Validating --> OutputNodes.W = LearnableParameter() : -> [10 x 64] -Validating --> h1.W = LearnableParameter() : -> [64 x 15 x 15 x 3] -Validating --> features = InputValue() : -> [32 x 32 x 3 x *] -Validating --> featOffs = LearnableParameter() : -> [1 x 1] -Validating --> featScaled = Minus (features, featOffs) : [32 x 32 x 3 x *], [1 x 1] -> [32 x 32 x 3 x *] -Validating --> sc = LearnableParameter() : -> [3 x 1] -Validating --> b = LearnableParameter() : -> [3 x 1] -Validating --> m = LearnableParameter() : -> [3 x 1] -Validating --> var = LearnableParameter() : -> [3 x 1] -Validating --> y = BatchNormalization (featScaled, sc, b, m, var) : [32 x 32 x 3 x *], [3 x 1], [3 x 1], [3 x 1], [3 x 1] -> [32 x 32 x 3 x *] -Validating --> conv1 = RectifiedLinear (y) : [32 x 32 x 3 x *] -> [32 x 32 x 3 x *] -Validating --> pool1 = MaxPooling (conv1) : [32 x 32 x 3 x *] -> [15 x 15 x 3 x *] -Validating --> h1.t = Times (h1.W, pool1) : [64 x 15 x 15 x 3], [15 x 15 x 3 x *] -> [64 x *] -Validating --> h1.b = LearnableParameter() : -> [64 x 1] -Validating --> h1.z = Plus (h1.t, h1.b) : [64 x *], [64 x 1] -> [64 x 1 x *] -Validating --> h1.y = RectifiedLinear (h1.z) : [64 x 1 x *] -> [64 x 1 x *] -Validating --> OutputNodes.t = Times (OutputNodes.W, h1.y) : [10 x 64], [64 x 1 x *] -> [10 x 1 x *] -Validating --> OutputNodes.b = LearnableParameter() : -> [10] -Validating --> OutputNodes.z = Plus (OutputNodes.t, OutputNodes.b) : [10 x 1 x *], [10] -> [10 x 1 x *] -Validating --> CE = CrossEntropyWithSoftmax (labels, OutputNodes.z) : [10 x *], [10 x 1 x *] -> [1] -Validating --> Err = ErrorPrediction (labels, OutputNodes.z) : [10 x *], [10 x 1 x *] -> [1] - -Validating network. 11 nodes to process in pass 2. - - -Validating network, final pass. - - +Microsoft::MSR::CNTK::GPUMatrix::SetGaussianRandomValue (GPU): creating curand object with seed 2, sizeof(ElemType)==4 +c: using cuDNN convolution engine for geometry: Input: 32 x 32 x 3, Output: 32 x 32 x 3, Kernel: 5 x 5 x 3, Map: 1 x 1 x 3, Stride: 1 x 1 x 3, Sharing: (1), AutoPad: (1), LowerPad: 0, UpperPad: 0. Using cuDNN batch normalization engine. - pool1: using cuDNN convolution engine for geometry: Input: 32 x 32 x 3, Output: 15 x 15 x 3, Kernel: 3 x 3 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1), AutoPad: (0), LowerPad: 0, UpperPad: 0. +01/04/2017 07:42:58: +Model has 24 nodes. Using GPU 0. - -11 out of 22 nodes do not share the minibatch layout with the input data. - -Post-processing network complete. - -08/23/2016 03:52:16: Created model with 22 nodes on GPU 0. - -08/23/2016 03:52:16: Training criterion node(s): -08/23/2016 03:52:16: CE = CrossEntropyWithSoftmax - -08/23/2016 03:52:16: Evaluation criterion node(s): -08/23/2016 03:52:16: Err = ErrorPrediction +01/04/2017 07:42:58: Training criterion: CE = CrossEntropyWithSoftmax +01/04/2017 07:42:58: Evaluation criterion: Err = ClassificationError Allocating matrices for forward and/or backward propagation. -Memory Sharing: Out of 37 matrices, 18 are shared as 8, and 19 are not shared. +Memory Sharing: Out of 41 matrices, 19 are shared as 7, and 22 are not shared. - { pool1 : [15 x 15 x 3 x *] - y : [32 x 32 x 3 x *] (gradient) } - { b : [3 x 1] (gradient) - conv1 : [32 x 32 x 3 x *] (gradient) - h1.t : [64 x *] } - { conv1 : [32 x 32 x 3 x *] - sc : [3 x 1] (gradient) } - { h1.W : [64 x 15 x 15 x 3] (gradient) - h1.z : [64 x 1 x *] } { OutputNodes.t : [10 x 1 x *] - h1.z : [64 x 1 x *] (gradient) - pool1 : [15 x 15 x 3 x *] (gradient) } - { h1.t : [64 x *] (gradient) - h1.y : [64 x 1 x *] } - { h1.b : [64 x 1] (gradient) - h1.y : [64 x 1 x *] (gradient) } + h1.z : [64 x 1 x *] } { OutputNodes.W : [10 x 64] (gradient) OutputNodes.z : [10 x 1 x *] (gradient) } + { OutputNodes.t : [10 x 1 x *] (gradient) + h1.W : [64 x 15 x 15 x 3] (gradient) + h1.z : [64 x 1 x *] (gradient) } + { c : [32 x 32 x 3 x *] (gradient) + conv1 : [32 x 32 x 3 x *] } + { h1.b : [64 x 1] (gradient) + h1.y : [64 x 1 x *] (gradient) } + { pool1 : [15 x 15 x 3 x *] + y : [32 x 32 x 3 x *] + y : [32 x 32 x 3 x *] (gradient) } + { conv1 : [32 x 32 x 3 x *] (gradient) + h1.t : [64 x *] + h1.t : [64 x *] (gradient) + h1.y : [64 x 1 x *] + sc : [3 x 1] (gradient) } -08/23/2016 03:52:16: Training 43920 parameters in 6 out of 6 parameter tensors and 15 nodes with gradient: +01/04/2017 07:42:58: Training 44145 parameters in 7 out of 7 parameter tensors and 17 nodes with gradient: -08/23/2016 03:52:16: Node 'OutputNodes.W' (LearnableParameter operation) : [10 x 64] -08/23/2016 03:52:16: Node 'OutputNodes.b' (LearnableParameter operation) : [10] -08/23/2016 03:52:16: Node 'b' (LearnableParameter operation) : [3 x 1] -08/23/2016 03:52:16: Node 'h1.W' (LearnableParameter operation) : [64 x 15 x 15 x 3] -08/23/2016 03:52:16: Node 'h1.b' (LearnableParameter operation) : [64 x 1] -08/23/2016 03:52:16: Node 'sc' (LearnableParameter operation) : [3 x 1] +01/04/2017 07:42:58: Node 'OutputNodes.W' (LearnableParameter operation) : [10 x 64] +01/04/2017 07:42:58: Node 'OutputNodes.b' (LearnableParameter operation) : [10] +01/04/2017 07:42:58: Node 'W' (LearnableParameter operation) : [3 x 75] +01/04/2017 07:42:58: Node 'b' (LearnableParameter operation) : [3 x 1] +01/04/2017 07:42:58: Node 'h1.W' (LearnableParameter operation) : [64 x 15 x 15 x 3] +01/04/2017 07:42:58: Node 'h1.b' (LearnableParameter operation) : [64 x 1] +01/04/2017 07:42:58: Node 'sc' (LearnableParameter operation) : [3 x 1] -08/23/2016 03:52:16: No PreCompute nodes found, or all already computed. Skipping pre-computation step. +01/04/2017 07:42:58: No PreCompute nodes found, or all already computed. Skipping pre-computation step. -08/23/2016 03:52:16: Starting Epoch 1: learning rate per sample = 0.000469 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 1: samples [0..1024] (first sequence at sample 0), worker rank 0, total workers 1 +01/04/2017 07:42:58: Starting Epoch 1: learning rate per sample = 0.000469 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/23/2016 03:52:16: Starting minibatch loop. -08/23/2016 03:52:20: Finished Epoch[ 1 of 2]: [Training] CE = 2.32012272 * 1024; Err = 0.89355469 * 1024; totalSamplesSeen = 1024; learningRatePerSample = 0.00046874999; epochTime=4.45523s -08/23/2016 03:52:20: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv.1' +01/04/2017 07:43:00: Starting minibatch loop. +01/04/2017 07:44:23: Finished Epoch[ 1 of 2]: [Training] CE = 2.27208638 * 1024; Err = 0.85546875 * 1024; totalSamplesSeen = 1024; learningRatePerSample = 0.00046874999; epochTime=84.4367s +01/04/2017 07:44:23: SGD: Saving checkpoint model 'C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv.1' -08/23/2016 03:52:20: Starting Epoch 2: learning rate per sample = 0.000469 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 2: samples [1024..2048] (first sequence at sample 1024), worker rank 0, total workers 1 +01/04/2017 07:44:23: Starting Epoch 2: learning rate per sample = 0.000469 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/23/2016 03:52:20: Starting minibatch loop. -08/23/2016 03:52:20: Finished Epoch[ 2 of 2]: [Training] CE = 2.24044609 * 1024; Err = 0.83984375 * 1024; totalSamplesSeen = 2048; learningRatePerSample = 0.00046874999; epochTime=0.037262s -08/23/2016 03:52:20: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv' -08/23/2016 03:52:20: CNTKCommandTrainEnd: Train +01/04/2017 07:44:23: Starting minibatch loop. +01/04/2017 07:44:23: Finished Epoch[ 2 of 2]: [Training] CE = 2.21359444 * 1024; Err = 0.80859375 * 1024; totalSamplesSeen = 2048; learningRatePerSample = 0.00046874999; epochTime=0.253057s +01/04/2017 07:44:23: SGD: Saving checkpoint model 'C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv' -08/23/2016 03:52:20: Action "train" complete. +01/04/2017 07:44:24: Action "train" complete. -08/23/2016 03:52:20: ############################################################################## -08/23/2016 03:52:20: # # -08/23/2016 03:52:20: # Action "test" # -08/23/2016 03:52:20: # # -08/23/2016 03:52:20: ############################################################################## +01/04/2017 07:44:24: ############################################################################## +01/04/2017 07:44:24: # # +01/04/2017 07:44:24: # Test command (test action) # +01/04/2017 07:44:24: # # +01/04/2017 07:44:24: ############################################################################## -INFO: y: initialized samplesSeen from mbCount when loading pre-CuDNNv5 model Post-processing network... 3 roots: CE = CrossEntropyWithSoftmax() - Err = ErrorPrediction() + Err = ClassificationError() OutputNodes.z = Plus() -Validating network. 22 nodes to process in pass 1. +Validating network. 24 nodes to process in pass 1. Validating --> labels = InputValue() : -> [10 x *1] Validating --> OutputNodes.W = LearnableParameter() : -> [10 x 64] Validating --> h1.W = LearnableParameter() : -> [64 x 15 x 15 x 3] +Validating --> W = LearnableParameter() : -> [3 x 75] Validating --> features = InputValue() : -> [32 x 32 x 3 x *1] Validating --> featOffs = LearnableParameter() : -> [1 x 1] Validating --> featScaled = Minus (features, featOffs) : [32 x 32 x 3 x *1], [1 x 1] -> [32 x 32 x 3 x *1] +Validating --> c = Convolution (W, featScaled) : [3 x 75], [32 x 32 x 3 x *1] -> [32 x 32 x 3 x *1] Validating --> sc = LearnableParameter() : -> [3 x 1] Validating --> b = LearnableParameter() : -> [3 x 1] Validating --> m = LearnableParameter() : -> [3 x 1] -Validating --> var = LearnableParameter() : -> [3 x 1] -Validating --> y = BatchNormalization (featScaled, sc, b, m, var) : [32 x 32 x 3 x *1], [3 x 1], [3 x 1], [3 x 1], [3 x 1] -> [32 x 32 x 3 x *1] +Validating --> v = LearnableParameter() : -> [3 x 1] +Validating --> y = BatchNormalization (c, sc, b, m, v) : [32 x 32 x 3 x *1], [3 x 1], [3 x 1], [3 x 1], [3 x 1] -> [32 x 32 x 3 x *1] Validating --> conv1 = RectifiedLinear (y) : [32 x 32 x 3 x *1] -> [32 x 32 x 3 x *1] Validating --> pool1 = MaxPooling (conv1) : [32 x 32 x 3 x *1] -> [15 x 15 x 3 x *1] Validating --> h1.t = Times (h1.W, pool1) : [64 x 15 x 15 x 3], [15 x 15 x 3 x *1] -> [64 x *1] @@ -461,20 +370,18 @@ Validating --> OutputNodes.t = Times (OutputNodes.W, h1.y) : [10 x 64], [64 x 1 Validating --> OutputNodes.b = LearnableParameter() : -> [10] Validating --> OutputNodes.z = Plus (OutputNodes.t, OutputNodes.b) : [10 x 1 x *1], [10] -> [10 x 1 x *1] Validating --> CE = CrossEntropyWithSoftmax (labels, OutputNodes.z) : [10 x *1], [10 x 1 x *1] -> [1] -Validating --> Err = ErrorPrediction (labels, OutputNodes.z) : [10 x *1], [10 x 1 x *1] -> [1] +Validating --> Err = ClassificationError (labels, OutputNodes.z) : [10 x *1], [10 x 1 x *1] -> [1] -Validating network. 11 nodes to process in pass 2. +Validating network. 12 nodes to process in pass 2. Validating network, final pass. - +c: using cuDNN convolution engine for geometry: Input: 32 x 32 x 3, Output: 32 x 32 x 3, Kernel: 5 x 5 x 3, Map: 1 x 1 x 3, Stride: 1 x 1 x 3, Sharing: (1), AutoPad: (1), LowerPad: 0, UpperPad: 0. Using cuDNN batch normalization engine. - pool1: using cuDNN convolution engine for geometry: Input: 32 x 32 x 3, Output: 15 x 15 x 3, Kernel: 3 x 3 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1), AutoPad: (0), LowerPad: 0, UpperPad: 0. -11 out of 22 nodes do not share the minibatch layout with the input data. Post-processing network complete. @@ -483,14 +390,23 @@ evalNodeNames are not specified, using all the default evalnodes and training cr Allocating matrices for forward and/or backward propagation. -Memory Sharing: Out of 22 matrices, 0 are shared as 0, and 22 are not shared. +Memory Sharing: Out of 24 matrices, 10 are shared as 2, and 14 are not shared. + { OutputNodes.z : [10 x 1 x *1] + c : [32 x 32 x 3 x *1] + conv1 : [32 x 32 x 3 x *1] + h1.t : [64 x *1] + h1.y : [64 x 1 x *1] } + { OutputNodes.t : [10 x 1 x *1] + featScaled : [32 x 32 x 3 x *1] + h1.z : [64 x 1 x *1] + pool1 : [15 x 15 x 3 x *1] + y : [32 x 32 x 3 x *1] } -BlockRandomizer::StartEpoch: epoch 1: samples [0..10000] (first sequence at sample 0), worker rank 0, total workers 1 -08/23/2016 03:52:22: Minibatch[1-500]: Err = 0.80987500 * 8000; CE = 2.19471007 * 8000 -08/23/2016 03:52:22: Minibatch[501-625]: Err = 0.82000000 * 2000; CE = 2.21734574 * 2000 -08/23/2016 03:52:22: Final Results: Minibatch[1-625]: Err = 0.81190000 * 10000; CE = 2.19923720 * 10000; perplexity = 9.01813187 +01/04/2017 07:44:43: Minibatch[1-500]: Err = 0.81237500 * 8000; CE = 2.18292270 * 8000 +01/04/2017 07:44:45: Minibatch[501-625]: Err = 0.81300000 * 2000; CE = 2.18584554 * 2000 +01/04/2017 07:44:45: Final Results: Minibatch[1-625]: Err = 0.81250000 * 10000; CE = 2.18350727 * 10000; perplexity = 8.87738714 -08/23/2016 03:52:22: Action "test" complete. +01/04/2017 07:44:45: Action "test" complete. -08/23/2016 03:52:22: __COMPLETED__ \ No newline at end of file +01/04/2017 07:44:45: __COMPLETED__ diff --git a/Tests/EndToEndTests/BatchNormalization/Spatial/CuDNN/baseline.txt b/Tests/EndToEndTests/BatchNormalization/Spatial/CuDNN/baseline.txt index 990d9f0f4acb..f5f5576f8642 100644 --- a/Tests/EndToEndTests/BatchNormalization/Spatial/CuDNN/baseline.txt +++ b/Tests/EndToEndTests/BatchNormalization/Spatial/CuDNN/baseline.txt @@ -1,60 +1,44 @@ CPU info: - CPU Model Name: Intel(R) Xeon(R) CPU E5-2630 v2 @ 2.60GHz + CPU Model Name: Intel(R) Xeon(R) CPU E5-2620 0 @ 2.00GHz Hardware threads: 24 - Total Memory: 268381192 kB + Total Memory: 33476276 kB ------------------------------------------------------------------- -=== Running /cygdrive/c/jenkins/workspace/CNTK-Test-Windows-W1/x64/release/cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.cntk currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn -------------------------------------------------------------------- -Build info: - - Built time: Aug 23 2016 03:29:14 - Last modified date: Mon Aug 22 21:29:13 2016 - Build type: Release - Build target: GPU - With 1bit-SGD: no - Math lib: mkl - CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5 - CUB_PATH: C:\src\cub-1.4.1 - CUDNN_PATH: C:\NVIDIA\cudnn-5.0\cuda - Build Branch: HEAD - Build SHA1: 66498cf414f4bbf6392730c2abfbef6a5eeb8f7b - Built by svcphil on LIANA-09-w - Build Path: c:\jenkins\workspace\CNTK-Build-Windows\Source\CNTK\ -------------------------------------------------------------------- -Changed current directory to C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -08/23/2016 03:52:13: ------------------------------------------------------------------- -08/23/2016 03:52:13: Build info: - -08/23/2016 03:52:13: Built time: Aug 23 2016 03:29:14 -08/23/2016 03:52:13: Last modified date: Mon Aug 22 21:29:13 2016 -08/23/2016 03:52:13: Build type: Release -08/23/2016 03:52:13: Build target: GPU -08/23/2016 03:52:13: With 1bit-SGD: no -08/23/2016 03:52:13: Math lib: mkl -08/23/2016 03:52:13: CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.5 -08/23/2016 03:52:13: CUB_PATH: C:\src\cub-1.4.1 -08/23/2016 03:52:13: CUDNN_PATH: C:\NVIDIA\cudnn-5.0\cuda -08/23/2016 03:52:13: Build Branch: HEAD -08/23/2016 03:52:13: Build SHA1: 66498cf414f4bbf6392730c2abfbef6a5eeb8f7b -08/23/2016 03:52:13: Built by svcphil on LIANA-09-w -08/23/2016 03:52:13: Build Path: c:\jenkins\workspace\CNTK-Build-Windows\Source\CNTK\ -08/23/2016 03:52:13: ------------------------------------------------------------------- -08/23/2016 03:52:14: ------------------------------------------------------------------- -08/23/2016 03:52:14: GPU info: - -08/23/2016 03:52:14: Device[0]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/23/2016 03:52:14: Device[1]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/23/2016 03:52:14: Device[2]: cores = 2880; computeCapability = 3.5; type = "GeForce GTX 780 Ti"; memory = 3072 MB -08/23/2016 03:52:14: ------------------------------------------------------------------- - -08/23/2016 03:52:14: Running on DPHAIM-25 at 2016/08/23 03:52:14 -08/23/2016 03:52:14: Command line: -C:\jenkins\workspace\CNTK-Test-Windows-W1\x64\release\cntk.exe configFile=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.cntk currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn - - - -08/23/2016 03:52:14: >>>>>>>>>>>>>>>>>>>> RAW CONFIG (VARIABLES NOT RESOLVED) >>>>>>>>>>>>>>>>>>>> -08/23/2016 03:52:14: RootDir = "." +=== Running /cygdrive/e/NetScale/CNTK/git_repos/git_cntkv2Library_5/x64/debug/cntk.exe configFile=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.cntk currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn +CNTK 2.0.beta7.0+ (amitaga/dataInteropFeatures 5ea312, Jan 3 2017 23:37:25) on Amitaga-Win-DT3 at 2017/01/04 07:42:50 + +E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\x64\debug\cntk.exe configFile=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.cntk currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn +Changed current directory to C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +01/04/2017 07:42:52: ------------------------------------------------------------------- +01/04/2017 07:42:52: Build info: + +01/04/2017 07:42:52: Built time: Jan 3 2017 23:37:25 +01/04/2017 07:42:52: Last modified date: Tue Jan 3 18:50:06 2017 +01/04/2017 07:42:52: Build type: Debug +01/04/2017 07:42:52: Build target: GPU +01/04/2017 07:42:52: With 1bit-SGD: yes +01/04/2017 07:42:52: With ASGD: no +01/04/2017 07:42:52: Math lib: mkl +01/04/2017 07:42:52: CUDA_PATH: C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v7.0 +01/04/2017 07:42:52: CUB_PATH: C:\cub-1.4.1 +01/04/2017 07:42:52: CUDNN_PATH: C:\cudnn-5.1 +01/04/2017 07:42:52: Build Branch: amitaga/dataInteropFeatures +01/04/2017 07:42:52: Build SHA1: 5ea3126709951e8b903eeffbd2c3bc1b6bf1db53 (modified) +01/04/2017 07:42:52: Built by amitaga on Amitaga-Win-DT3 +01/04/2017 07:42:52: Build Path: E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Source\CNTK\ +01/04/2017 07:42:52: MPI distribution: msmpi +01/04/2017 07:42:52: MPI version: 7.0.12437.6 +01/04/2017 07:42:52: ------------------------------------------------------------------- +01/04/2017 07:42:53: ------------------------------------------------------------------- +01/04/2017 07:42:53: GPU info: + +01/04/2017 07:42:53: Device[0]: cores = 1536; computeCapability = 3.0; type = "GeForce GTX 690"; memory = 2048 MB +01/04/2017 07:42:53: Device[1]: cores = 384; computeCapability = 3.0; type = "GeForce GTX 650"; memory = 1024 MB +01/04/2017 07:42:53: Device[2]: cores = 1536; computeCapability = 3.0; type = "GeForce GTX 690"; memory = 2048 MB +01/04/2017 07:42:53: ------------------------------------------------------------------- + +Configuration, Raw: + +01/04/2017 07:42:53: RootDir = "." ConfigDir = "$RootDir$" DataDir = "$RootDir$" OutputDir = "$RootDir$/Output" @@ -117,24 +101,24 @@ Test = [ ] ] ] -currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu -DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial -OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu +currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu +DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial +OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn -08/23/2016 03:52:14: <<<<<<<<<<<<<<<<<<<< RAW CONFIG (VARIABLES NOT RESOLVED) <<<<<<<<<<<<<<<<<<<< -08/23/2016 03:52:14: >>>>>>>>>>>>>>>>>>>> RAW CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>> -08/23/2016 03:52:14: RootDir = "." +Configuration After Variable Resolution: + +01/04/2017 07:42:53: RootDir = "." ConfigDir = "." DataDir = "." OutputDir = "./Output" -ModelDir = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models" -ndlMacros = "C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/Macros.ndl" +ModelDir = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models" +ndlMacros = "E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/Macros.ndl" precision = "float" deviceId = 0 imageLayout = "cudnn" @@ -145,9 +129,9 @@ traceLevel = 1 numMBsToShowResult = 500 Train = [ action = "train" - modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv" + modelPath = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv" NDLNetworkBuilder = [ - networkDescription = "C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.ndl" + networkDescription = "E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.ndl" ] SGD = [ epochSize = 1024 @@ -160,7 +144,7 @@ Train = [ ] reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData/Train_cntk_text.txt" + file = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData/Train_cntk_text.txt" input = [ features = [ dim = 3072 @@ -175,11 +159,11 @@ Train = [ ] Test = [ action = "test" - modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv" + modelPath = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv" minibatchSize = 16 reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData/Test_cntk_text.txt" + file = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData/Test_cntk_text.txt" input = [ features = [ dim = 3072 @@ -192,40 +176,40 @@ Test = [ ] ] ] -currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu -DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial -OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu +currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu +DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial +OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu DeviceId=0 timestamping=true batchNormalizationEngine=cudnn -08/23/2016 03:52:14: <<<<<<<<<<<<<<<<<<<< RAW CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<< -08/23/2016 03:52:14: >>>>>>>>>>>>>>>>>>>> PROCESSED CONFIG WITH ALL VARIABLES RESOLVED >>>>>>>>>>>>>>>>>>>> +Configuration After Processing and Variable Resolution: + configparameters: 02_BatchNormConv.cntk:batchNormalizationEngine=cudnn configparameters: 02_BatchNormConv.cntk:command=Train:Test -configparameters: 02_BatchNormConv.cntk:ConfigDir=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial -configparameters: 02_BatchNormConv.cntk:currentDirectory=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData -configparameters: 02_BatchNormConv.cntk:DataDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData +configparameters: 02_BatchNormConv.cntk:ConfigDir=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial +configparameters: 02_BatchNormConv.cntk:currentDirectory=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData +configparameters: 02_BatchNormConv.cntk:DataDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData configparameters: 02_BatchNormConv.cntk:deviceId=0 configparameters: 02_BatchNormConv.cntk:imageLayout=cudnn configparameters: 02_BatchNormConv.cntk:initOnCPUOnly=true -configparameters: 02_BatchNormConv.cntk:ModelDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models -configparameters: 02_BatchNormConv.cntk:ndlMacros=C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/Macros.ndl +configparameters: 02_BatchNormConv.cntk:ModelDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models +configparameters: 02_BatchNormConv.cntk:ndlMacros=E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/Macros.ndl configparameters: 02_BatchNormConv.cntk:numMBsToShowResult=500 -configparameters: 02_BatchNormConv.cntk:OutputDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu +configparameters: 02_BatchNormConv.cntk:OutputDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu configparameters: 02_BatchNormConv.cntk:precision=float configparameters: 02_BatchNormConv.cntk:RootDir=. -configparameters: 02_BatchNormConv.cntk:RunDir=C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu +configparameters: 02_BatchNormConv.cntk:RunDir=C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu configparameters: 02_BatchNormConv.cntk:Test=[ action = "test" - modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv" + modelPath = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv" minibatchSize = 16 reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData/Test_cntk_text.txt" + file = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData/Test_cntk_text.txt" input = [ features = [ dim = 3072 @@ -243,9 +227,9 @@ configparameters: 02_BatchNormConv.cntk:timestamping=true configparameters: 02_BatchNormConv.cntk:traceLevel=1 configparameters: 02_BatchNormConv.cntk:Train=[ action = "train" - modelPath = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv" + modelPath = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv" NDLNetworkBuilder = [ - networkDescription = "C:\jenkins\workspace\CNTK-Test-Windows-W1\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.ndl" + networkDescription = "E:\NetScale\CNTK\git_repos\git_cntkv2Library_5\Tests\EndToEndTests\BatchNormalization\Spatial/02_BatchNormConv.ndl" ] SGD = [ epochSize = 1024 @@ -258,7 +242,7 @@ configparameters: 02_BatchNormConv.cntk:Train=[ ] reader = [ readerType = "CNTKTextFormatReader" - file = "C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu\TestData/Train_cntk_text.txt" + file = "C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu\TestData/Train_cntk_text.txt" input = [ features = [ dim = 3072 @@ -272,185 +256,110 @@ configparameters: 02_BatchNormConv.cntk:Train=[ ] ] -08/23/2016 03:52:14: <<<<<<<<<<<<<<<<<<<< PROCESSED CONFIG WITH ALL VARIABLES RESOLVED <<<<<<<<<<<<<<<<<<<< -08/23/2016 03:52:14: Commands: Train Test -08/23/2016 03:52:14: Precision = "float" -08/23/2016 03:52:14: CNTKModelPath: C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv -08/23/2016 03:52:14: CNTKCommandTrainInfo: Train : 2 -08/23/2016 03:52:14: CNTKCommandTrainInfo: CNTKNoMoreCommands_Total : 2 - -08/23/2016 03:52:14: ############################################################################## -08/23/2016 03:52:14: # # -08/23/2016 03:52:14: # Action "train" # -08/23/2016 03:52:14: # # -08/23/2016 03:52:14: ############################################################################## +01/04/2017 07:42:53: Commands: Train Test +01/04/2017 07:42:53: precision = "float" -08/23/2016 03:52:14: CNTKCommandTrainBegin: Train +01/04/2017 07:42:53: ############################################################################## +01/04/2017 07:42:53: # # +01/04/2017 07:42:53: # Train command (train action) # +01/04/2017 07:42:53: # # +01/04/2017 07:42:53: ############################################################################## -08/23/2016 03:52:14: Creating virgin network. +01/04/2017 07:42:53: +Creating virgin network. NDLBuilder Using GPU 0 -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 0.000000. -Node 'b' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'sc' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'm' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'var' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'h1.W' (LearnableParameter operation): Initializing Parameter[64 x 15 x 15 x 3] <- 0.000000. -Node 'h1.b' (LearnableParameter operation): Initializing Parameter[64 x 1] <- 0.000000. -Node 'OutputNodes.W' (LearnableParameter operation): Initializing Parameter[10 x 64] <- 0.000000. -Node 'OutputNodes.b' (LearnableParameter operation): Initializing Parameter[10] <- 0.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'featOffs' (LearnableParameter operation): Initializing Parameter[1 x 1] <- 128.000000. -Node 'b' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'sc' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 1.000000. -Node 'm' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'var' (LearnableParameter operation): Initializing Parameter[3 x 1] <- 0.000000. -Node 'h1.W' (LearnableParameter operation): Initializing Parameter[64 x 15 x 15 x 3] <- gaussian(seed=1, range=0.007698*12.000000, onCPU=false). -Microsoft::MSR::CNTK::GPUMatrix::SetGaussianRandomValue (GPU): creating curand object with seed 1, sizeof(ElemType)==4 -Node 'h1.b' (LearnableParameter operation): Initializing Parameter[64 x 1] <- 0.000000. -Node 'OutputNodes.W' (LearnableParameter operation): Initializing Parameter[10 x 64] <- gaussian(seed=2, range=0.025000*1.500000, onCPU=false). -Node 'OutputNodes.b' (LearnableParameter operation): Initializing Parameter[10] <- 0.000000. - -Post-processing network... - -3 roots: - CE = CrossEntropyWithSoftmax() - Err = ErrorPrediction() - OutputNodes.z = Plus() - -Validating network. 22 nodes to process in pass 1. - -Validating --> labels = InputValue() : -> [10 x *] -Validating --> OutputNodes.W = LearnableParameter() : -> [10 x 64] -Validating --> h1.W = LearnableParameter() : -> [64 x 15 x 15 x 3] -Validating --> features = InputValue() : -> [32 x 32 x 3 x *] -Validating --> featOffs = LearnableParameter() : -> [1 x 1] -Validating --> featScaled = Minus (features, featOffs) : [32 x 32 x 3 x *], [1 x 1] -> [32 x 32 x 3 x *] -Validating --> sc = LearnableParameter() : -> [3 x 1] -Validating --> b = LearnableParameter() : -> [3 x 1] -Validating --> m = LearnableParameter() : -> [3 x 1] -Validating --> var = LearnableParameter() : -> [3 x 1] -Validating --> y = BatchNormalization (featScaled, sc, b, m, var) : [32 x 32 x 3 x *], [3 x 1], [3 x 1], [3 x 1], [3 x 1] -> [32 x 32 x 3 x *] -Validating --> conv1 = RectifiedLinear (y) : [32 x 32 x 3 x *] -> [32 x 32 x 3 x *] -Validating --> pool1 = MaxPooling (conv1) : [32 x 32 x 3 x *] -> [15 x 15 x 3 x *] -Validating --> h1.t = Times (h1.W, pool1) : [64 x 15 x 15 x 3], [15 x 15 x 3 x *] -> [64 x *] -Validating --> h1.b = LearnableParameter() : -> [64 x 1] -Validating --> h1.z = Plus (h1.t, h1.b) : [64 x *], [64 x 1] -> [64 x 1 x *] -Validating --> h1.y = RectifiedLinear (h1.z) : [64 x 1 x *] -> [64 x 1 x *] -Validating --> OutputNodes.t = Times (OutputNodes.W, h1.y) : [10 x 64], [64 x 1 x *] -> [10 x 1 x *] -Validating --> OutputNodes.b = LearnableParameter() : -> [10] -Validating --> OutputNodes.z = Plus (OutputNodes.t, OutputNodes.b) : [10 x 1 x *], [10] -> [10 x 1 x *] -Validating --> CE = CrossEntropyWithSoftmax (labels, OutputNodes.z) : [10 x *], [10 x 1 x *] -> [1] -Validating --> Err = ErrorPrediction (labels, OutputNodes.z) : [10 x *], [10 x 1 x *] -> [1] - -Validating network. 11 nodes to process in pass 2. - - -Validating network, final pass. - - +Microsoft::MSR::CNTK::GPUMatrix::SetGaussianRandomValue (GPU): creating curand object with seed 2, sizeof(ElemType)==4 +c: using cuDNN convolution engine for geometry: Input: 32 x 32 x 3, Output: 32 x 32 x 3, Kernel: 5 x 5 x 3, Map: 1 x 1 x 3, Stride: 1 x 1 x 3, Sharing: (1), AutoPad: (1), LowerPad: 0, UpperPad: 0. Using cuDNN batch normalization engine. - pool1: using cuDNN convolution engine for geometry: Input: 32 x 32 x 3, Output: 15 x 15 x 3, Kernel: 3 x 3 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1), AutoPad: (0), LowerPad: 0, UpperPad: 0. +01/04/2017 07:42:58: +Model has 24 nodes. Using GPU 0. - -11 out of 22 nodes do not share the minibatch layout with the input data. - -Post-processing network complete. - -08/23/2016 03:52:16: Created model with 22 nodes on GPU 0. - -08/23/2016 03:52:16: Training criterion node(s): -08/23/2016 03:52:16: CE = CrossEntropyWithSoftmax - -08/23/2016 03:52:16: Evaluation criterion node(s): -08/23/2016 03:52:16: Err = ErrorPrediction +01/04/2017 07:42:58: Training criterion: CE = CrossEntropyWithSoftmax +01/04/2017 07:42:58: Evaluation criterion: Err = ClassificationError Allocating matrices for forward and/or backward propagation. -Memory Sharing: Out of 37 matrices, 18 are shared as 8, and 19 are not shared. +Memory Sharing: Out of 41 matrices, 19 are shared as 7, and 22 are not shared. - { pool1 : [15 x 15 x 3 x *] - y : [32 x 32 x 3 x *] (gradient) } - { b : [3 x 1] (gradient) - conv1 : [32 x 32 x 3 x *] (gradient) - h1.t : [64 x *] } - { conv1 : [32 x 32 x 3 x *] - sc : [3 x 1] (gradient) } - { h1.W : [64 x 15 x 15 x 3] (gradient) - h1.z : [64 x 1 x *] } { OutputNodes.t : [10 x 1 x *] - h1.z : [64 x 1 x *] (gradient) - pool1 : [15 x 15 x 3 x *] (gradient) } - { h1.t : [64 x *] (gradient) - h1.y : [64 x 1 x *] } - { h1.b : [64 x 1] (gradient) - h1.y : [64 x 1 x *] (gradient) } + h1.z : [64 x 1 x *] } { OutputNodes.W : [10 x 64] (gradient) OutputNodes.z : [10 x 1 x *] (gradient) } + { OutputNodes.t : [10 x 1 x *] (gradient) + h1.W : [64 x 15 x 15 x 3] (gradient) + h1.z : [64 x 1 x *] (gradient) } + { c : [32 x 32 x 3 x *] (gradient) + conv1 : [32 x 32 x 3 x *] } + { h1.b : [64 x 1] (gradient) + h1.y : [64 x 1 x *] (gradient) } + { pool1 : [15 x 15 x 3 x *] + y : [32 x 32 x 3 x *] + y : [32 x 32 x 3 x *] (gradient) } + { conv1 : [32 x 32 x 3 x *] (gradient) + h1.t : [64 x *] + h1.t : [64 x *] (gradient) + h1.y : [64 x 1 x *] + sc : [3 x 1] (gradient) } -08/23/2016 03:52:16: Training 43920 parameters in 6 out of 6 parameter tensors and 15 nodes with gradient: +01/04/2017 07:42:58: Training 44145 parameters in 7 out of 7 parameter tensors and 17 nodes with gradient: -08/23/2016 03:52:16: Node 'OutputNodes.W' (LearnableParameter operation) : [10 x 64] -08/23/2016 03:52:16: Node 'OutputNodes.b' (LearnableParameter operation) : [10] -08/23/2016 03:52:16: Node 'b' (LearnableParameter operation) : [3 x 1] -08/23/2016 03:52:16: Node 'h1.W' (LearnableParameter operation) : [64 x 15 x 15 x 3] -08/23/2016 03:52:16: Node 'h1.b' (LearnableParameter operation) : [64 x 1] -08/23/2016 03:52:16: Node 'sc' (LearnableParameter operation) : [3 x 1] +01/04/2017 07:42:58: Node 'OutputNodes.W' (LearnableParameter operation) : [10 x 64] +01/04/2017 07:42:58: Node 'OutputNodes.b' (LearnableParameter operation) : [10] +01/04/2017 07:42:58: Node 'W' (LearnableParameter operation) : [3 x 75] +01/04/2017 07:42:58: Node 'b' (LearnableParameter operation) : [3 x 1] +01/04/2017 07:42:58: Node 'h1.W' (LearnableParameter operation) : [64 x 15 x 15 x 3] +01/04/2017 07:42:58: Node 'h1.b' (LearnableParameter operation) : [64 x 1] +01/04/2017 07:42:58: Node 'sc' (LearnableParameter operation) : [3 x 1] -08/23/2016 03:52:16: No PreCompute nodes found, or all already computed. Skipping pre-computation step. +01/04/2017 07:42:58: No PreCompute nodes found, or all already computed. Skipping pre-computation step. -08/23/2016 03:52:16: Starting Epoch 1: learning rate per sample = 0.000469 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 1: samples [0..1024] (first sequence at sample 0), worker rank 0, total workers 1 +01/04/2017 07:42:58: Starting Epoch 1: learning rate per sample = 0.000469 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/23/2016 03:52:16: Starting minibatch loop. -08/23/2016 03:52:20: Finished Epoch[ 1 of 2]: [Training] CE = 2.32012272 * 1024; Err = 0.89355469 * 1024; totalSamplesSeen = 1024; learningRatePerSample = 0.00046874999; epochTime=4.45523s -08/23/2016 03:52:20: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv.1' +01/04/2017 07:43:00: Starting minibatch loop. +01/04/2017 07:44:23: Finished Epoch[ 1 of 2]: [Training] CE = 2.27208638 * 1024; Err = 0.85546875 * 1024; totalSamplesSeen = 1024; learningRatePerSample = 0.00046874999; epochTime=84.4367s +01/04/2017 07:44:23: SGD: Saving checkpoint model 'C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv.1' -08/23/2016 03:52:20: Starting Epoch 2: learning rate per sample = 0.000469 effective momentum = 0.000000 momentum as time constant = 0.0 samples -BlockRandomizer::StartEpoch: epoch 2: samples [1024..2048] (first sequence at sample 1024), worker rank 0, total workers 1 +01/04/2017 07:44:23: Starting Epoch 2: learning rate per sample = 0.000469 effective momentum = 0.000000 momentum as time constant = 0.0 samples -08/23/2016 03:52:20: Starting minibatch loop. -08/23/2016 03:52:20: Finished Epoch[ 2 of 2]: [Training] CE = 2.24044609 * 1024; Err = 0.83984375 * 1024; totalSamplesSeen = 2048; learningRatePerSample = 0.00046874999; epochTime=0.037262s -08/23/2016 03:52:20: SGD: Saving checkpoint model 'C:\Users\svcphil\AppData\Local\Temp\cntk-test-20160823035118.956998\BatchNormalization\Spatial_CuDNN@release_gpu/Models/02_BatchNormConv' -08/23/2016 03:52:20: CNTKCommandTrainEnd: Train +01/04/2017 07:44:23: Starting minibatch loop. +01/04/2017 07:44:23: Finished Epoch[ 2 of 2]: [Training] CE = 2.21359444 * 1024; Err = 0.80859375 * 1024; totalSamplesSeen = 2048; learningRatePerSample = 0.00046874999; epochTime=0.253057s +01/04/2017 07:44:23: SGD: Saving checkpoint model 'C:\cygwin64\tmp\cntk-test-20170103234236.850903\BatchNormalization\Spatial_CuDNN@debug_gpu/Models/02_BatchNormConv' -08/23/2016 03:52:20: Action "train" complete. +01/04/2017 07:44:24: Action "train" complete. -08/23/2016 03:52:20: ############################################################################## -08/23/2016 03:52:20: # # -08/23/2016 03:52:20: # Action "test" # -08/23/2016 03:52:20: # # -08/23/2016 03:52:20: ############################################################################## +01/04/2017 07:44:24: ############################################################################## +01/04/2017 07:44:24: # # +01/04/2017 07:44:24: # Test command (test action) # +01/04/2017 07:44:24: # # +01/04/2017 07:44:24: ############################################################################## -INFO: y: initialized samplesSeen from mbCount when loading pre-CuDNNv5 model Post-processing network... 3 roots: CE = CrossEntropyWithSoftmax() - Err = ErrorPrediction() + Err = ClassificationError() OutputNodes.z = Plus() -Validating network. 22 nodes to process in pass 1. +Validating network. 24 nodes to process in pass 1. Validating --> labels = InputValue() : -> [10 x *1] Validating --> OutputNodes.W = LearnableParameter() : -> [10 x 64] Validating --> h1.W = LearnableParameter() : -> [64 x 15 x 15 x 3] +Validating --> W = LearnableParameter() : -> [3 x 75] Validating --> features = InputValue() : -> [32 x 32 x 3 x *1] Validating --> featOffs = LearnableParameter() : -> [1 x 1] Validating --> featScaled = Minus (features, featOffs) : [32 x 32 x 3 x *1], [1 x 1] -> [32 x 32 x 3 x *1] +Validating --> c = Convolution (W, featScaled) : [3 x 75], [32 x 32 x 3 x *1] -> [32 x 32 x 3 x *1] Validating --> sc = LearnableParameter() : -> [3 x 1] Validating --> b = LearnableParameter() : -> [3 x 1] Validating --> m = LearnableParameter() : -> [3 x 1] -Validating --> var = LearnableParameter() : -> [3 x 1] -Validating --> y = BatchNormalization (featScaled, sc, b, m, var) : [32 x 32 x 3 x *1], [3 x 1], [3 x 1], [3 x 1], [3 x 1] -> [32 x 32 x 3 x *1] +Validating --> v = LearnableParameter() : -> [3 x 1] +Validating --> y = BatchNormalization (c, sc, b, m, v) : [32 x 32 x 3 x *1], [3 x 1], [3 x 1], [3 x 1], [3 x 1] -> [32 x 32 x 3 x *1] Validating --> conv1 = RectifiedLinear (y) : [32 x 32 x 3 x *1] -> [32 x 32 x 3 x *1] Validating --> pool1 = MaxPooling (conv1) : [32 x 32 x 3 x *1] -> [15 x 15 x 3 x *1] Validating --> h1.t = Times (h1.W, pool1) : [64 x 15 x 15 x 3], [15 x 15 x 3 x *1] -> [64 x *1] @@ -461,20 +370,18 @@ Validating --> OutputNodes.t = Times (OutputNodes.W, h1.y) : [10 x 64], [64 x 1 Validating --> OutputNodes.b = LearnableParameter() : -> [10] Validating --> OutputNodes.z = Plus (OutputNodes.t, OutputNodes.b) : [10 x 1 x *1], [10] -> [10 x 1 x *1] Validating --> CE = CrossEntropyWithSoftmax (labels, OutputNodes.z) : [10 x *1], [10 x 1 x *1] -> [1] -Validating --> Err = ErrorPrediction (labels, OutputNodes.z) : [10 x *1], [10 x 1 x *1] -> [1] +Validating --> Err = ClassificationError (labels, OutputNodes.z) : [10 x *1], [10 x 1 x *1] -> [1] -Validating network. 11 nodes to process in pass 2. +Validating network. 12 nodes to process in pass 2. Validating network, final pass. - +c: using cuDNN convolution engine for geometry: Input: 32 x 32 x 3, Output: 32 x 32 x 3, Kernel: 5 x 5 x 3, Map: 1 x 1 x 3, Stride: 1 x 1 x 3, Sharing: (1), AutoPad: (1), LowerPad: 0, UpperPad: 0. Using cuDNN batch normalization engine. - pool1: using cuDNN convolution engine for geometry: Input: 32 x 32 x 3, Output: 15 x 15 x 3, Kernel: 3 x 3 x 1, Map: 1, Stride: 2 x 2 x 1, Sharing: (1), AutoPad: (0), LowerPad: 0, UpperPad: 0. -11 out of 22 nodes do not share the minibatch layout with the input data. Post-processing network complete. @@ -483,14 +390,23 @@ evalNodeNames are not specified, using all the default evalnodes and training cr Allocating matrices for forward and/or backward propagation. -Memory Sharing: Out of 22 matrices, 0 are shared as 0, and 22 are not shared. +Memory Sharing: Out of 24 matrices, 10 are shared as 2, and 14 are not shared. + { OutputNodes.z : [10 x 1 x *1] + c : [32 x 32 x 3 x *1] + conv1 : [32 x 32 x 3 x *1] + h1.t : [64 x *1] + h1.y : [64 x 1 x *1] } + { OutputNodes.t : [10 x 1 x *1] + featScaled : [32 x 32 x 3 x *1] + h1.z : [64 x 1 x *1] + pool1 : [15 x 15 x 3 x *1] + y : [32 x 32 x 3 x *1] } -BlockRandomizer::StartEpoch: epoch 1: samples [0..10000] (first sequence at sample 0), worker rank 0, total workers 1 -08/23/2016 03:52:22: Minibatch[1-500]: Err = 0.80987500 * 8000; CE = 2.19471007 * 8000 -08/23/2016 03:52:22: Minibatch[501-625]: Err = 0.82000000 * 2000; CE = 2.21734574 * 2000 -08/23/2016 03:52:22: Final Results: Minibatch[1-625]: Err = 0.81190000 * 10000; CE = 2.19923720 * 10000; perplexity = 9.01813187 +01/04/2017 07:44:43: Minibatch[1-500]: Err = 0.81237500 * 8000; CE = 2.18292270 * 8000 +01/04/2017 07:44:45: Minibatch[501-625]: Err = 0.81300000 * 2000; CE = 2.18584554 * 2000 +01/04/2017 07:44:45: Final Results: Minibatch[1-625]: Err = 0.81250000 * 10000; CE = 2.18350727 * 10000; perplexity = 8.87738714 -08/23/2016 03:52:22: Action "test" complete. +01/04/2017 07:44:45: Action "test" complete. -08/23/2016 03:52:22: __COMPLETED__ \ No newline at end of file +01/04/2017 07:44:45: __COMPLETED__ diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index 79dda3215e58..26c7a2ddbf2b 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -417,7 +417,9 @@ %ignore CNTK::Internal::ForceDeterministicAlgorithms(); %ignore CNTK::Internal::SetFixedRandomSeed(unsigned long fixedRandomSeed); %ignore CNTK::Internal::EnableForwardValuesSharing(); +%ignore CNTK::Internal::DisableForwardValuesSharing(); %ignore CNTK::Internal::EnableHyperMemoryCompress(); +%ignore CNTK::Internal::DisableHyperMemoryCompress(); %ignore CNTK::Internal::AreEquivalent(const ::CNTK::FunctionPtr& f1, const ::CNTK::FunctionPtr& f2); %ignore CNTK::Internal::AreEquivalent(const ::CNTK::Variable& v1, const ::CNTK::Variable& v2, bool allowParameterAndConstantsEquivalence = false); %ignore CNTK::Internal::AreEqual(const ::CNTK::NDArrayView& view1, const ::CNTK::NDArrayView& view2, double relativeTolerance = 0.0, double absoluteTolerance = 0.0); From 439fd4eb493c5b5570c66a0a1631b6bad3c5086a Mon Sep 17 00:00:00 2001 From: Cha Zhang Date: Tue, 3 Jan 2017 17:37:03 -0800 Subject: [PATCH 010/120] Bug fix for asymmetric padding for convolution. Added a test too. --- Source/Math/CuDnnConvolutionEngine.cu | 69 ++++++++++++------- bindings/python/cntk/ops/tests/kernel_test.py | 41 +++++++++++ 2 files changed, 87 insertions(+), 23 deletions(-) diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu index c2ba9b6054ed..55f0d2f14fa7 100644 --- a/Source/Math/CuDnnConvolutionEngine.cu +++ b/Source/Math/CuDnnConvolutionEngine.cu @@ -22,7 +22,9 @@ const char* CudaErrString(cudnnStatus_t x) #define TENSOR_FORMAT CUDNN_TENSOR_NCHW #define FILTER_FORMAT CUDNN_TENSOR_NCHW -namespace Microsoft { namespace MSR { namespace CNTK { +namespace Microsoft { +namespace MSR { +namespace CNTK { static bool IsGpu(DEVICEID_TYPE deviceId) { @@ -90,8 +92,8 @@ public: } SmallVector upscale(stride.size(), 1); CUDNN_CALL(cudnnSetConvolutionNdDescriptor(m_conv, (int)stride.size(), pad.data(), - stride.data(), upscale.data(), - CUDNN_CROSS_CORRELATION, dataType)); + stride.data(), upscale.data(), + CUDNN_CROSS_CORRELATION, dataType)); } ~CuDnnConv() @@ -139,9 +141,9 @@ public: // Must use CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING to get the same results as in reference engine. CUDNN_CALL(cudnnSetPoolingNdDescriptor(m_pool, - kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, - CUDNN_PROPAGATE_NAN, - (int)dims.size(), dims.data(), pad.data(), stride.data())); + kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, + CUDNN_PROPAGATE_NAN, + (int)dims.size(), dims.data(), pad.data(), stride.data())); } ~CuDnnPool() @@ -183,7 +185,9 @@ public: { } - virtual bool ImplementsGradientOverwriteOptimization() const override { return true; } + virtual bool ImplementsGradientOverwriteOptimization() const override { + return true; + } protected: using Base::m_geometry; @@ -204,8 +208,8 @@ protected: { if (m_kernelT == nullptr) { - m_kernelT = std::make_unique(*m_geometry, m_dataType), - m_conv = std::make_unique(*m_geometry, m_dataType); + m_kernelT = std::make_unique(*m_geometry, m_dataType), + m_conv = std::make_unique(*m_geometry, m_dataType); } } @@ -219,7 +223,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -267,7 +271,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -284,7 +288,7 @@ protected: workspace.Resize((m_backDataAlgo.Algo.memory + sizeof(ElemType) - 1) / sizeof(ElemType), 1); // Compute gradients with respect to the output tensor (data). CUDNN_CALL(cudnnConvolutionBackwardData(*m_cudnn, &C::One, *m_kernelT, ptr(kernel), m_outT, ptr(srcGrad), *m_conv, m_backDataAlgo.Algo.algo, - ptr(workspace), m_backDataAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, m_inT, ptr(grad))); + ptr(workspace), m_backDataAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, m_inT, ptr(grad))); workspace.Resize(0, 0); } @@ -298,7 +302,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -315,7 +319,7 @@ protected: workspace.Resize((m_backFiltAlgo.Algo.memory + sizeof(ElemType) - 1) / sizeof(ElemType), 1); // Compute gradients with respect to the output tensor (data). CUDNN_CALL(cudnnConvolutionBackwardFilter(*m_cudnn, &C::One, m_inT, ptr(in), m_outT, ptr(srcGrad), *m_conv, m_backFiltAlgo.Algo.algo, - ptr(workspace), m_backFiltAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, *m_kernelT, ptr(kernelGrad))); + ptr(workspace), m_backFiltAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, *m_kernelT, ptr(kernelGrad))); workspace.Resize(0, 0); } @@ -339,7 +343,7 @@ protected: m_inT.UpdateBatchSize(batchSize); m_outT.UpdateBatchSize(batchSize); CUDNN_CALL(cudnnPoolingBackward(*m_cudnn, *(m_pool), &C::One, m_outT, ptr(out), m_outT, ptr(srcGrad), - m_inT, ptr(in), &C::One, m_inT, ptr(grad))); + m_inT, ptr(in), &C::One, m_inT, ptr(grad))); } void MaxUnpoolingCore(const Mat& out, const Mat& poolIn, Mat& in) override @@ -393,7 +397,7 @@ private: size_t maxMem = m_maxTempMemSizeInSamples == 0 ? (std::numeric_limits::max)() : inputSampleSize * m_maxTempMemSizeInSamples * sizeof(ElemType); // Find best (fastest) algorithm which satisfies workspace requirements. auto res = std::find_if(algoPerf, algoPerf + calgo, - [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); + [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); if (res == algoPerf + calgo) RuntimeError("cuDNN could not find suitable algorithm for the current convolution configuration."); @@ -405,8 +409,8 @@ private: // Find fastest algorithm that does NOT require workspace. It is used as a fallback algo in Forward function. // Currently all Forward algorithms are deterministic, so no need for checking. - res = std::find_if(algoPerf, algoPerf + calgo, - [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); + res = std::find_if(algoPerf, algoPerf + calgo, + [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); if (res == algoPerf + calgo) { // In theory, this should never happen. @@ -497,14 +501,33 @@ bool CuDnnConvolutionEngineFactory::IsSupported(DEVICEID_TYPE deviceId const auto& kernel = geometry->KernelShape(); const auto& sharing = geometry->Sharing(); const auto& mapCount = geometry->MapCount(); + + const auto& inputRank = input.GetRank(); + const auto& kernelRank = kernel.GetRank(); + const auto& mapRank = mapCount.GetRank(); // cuDNN supports 2D and 3D convolutions at the moment with full sharing. // In case map count size > 1, then it should have all ones except last dimension. // If pooling is requested, then cuDNN supports only 2D/3D inputs and 2D pooling kernels. - return (input.GetRank() <= 4 && - std::find(begin(sharing), end(sharing), false) == sharing.end() && - mapCount.GetNumElements() == mapCount[mapCount.GetRank() - 1] && - (poolKind == PoolKind::None || - input.GetRank() <= 3 && (kernel.GetRank() < 3 || kernel[2] == 1))); + bool retVal = (inputRank <= 4 && + std::find(begin(sharing), end(sharing), false) == sharing.end() && + mapCount.GetNumElements() == mapCount[mapRank - 1] && + (poolKind == PoolKind::None || + inputRank <= 3 && (kernelRank < 3 || kernel[2] == 1))); + + // cuDNN as of version 8.0 does not handle asymmetric padding correctly. We need to + // detect asymmetric padding (either by auto-padding or by specifying lowerPad/upperPad) and choose + // the reference convolution implementation instead + const auto& autopad = geometry->AutoPad(); + const auto& lowerpad = geometry->LowerPad(); + const auto& upperpad = geometry->UpperPad(); + for (int i = 0; i < kernelRank; i++) + { + if (autopad[i]) + retVal = retVal && (kernel[i] % 2 != 0); // make sure kernel size is odd + else + retVal = retVal && (lowerpad[i] == upperpad[i]); // make sure lower and upper padding are equal + } + return retVal; } template class CuDnnConvolutionEngineFactory; diff --git a/bindings/python/cntk/ops/tests/kernel_test.py b/bindings/python/cntk/ops/tests/kernel_test.py index 9f8e1a23854d..afe13057f02a 100644 --- a/bindings/python/cntk/ops/tests/kernel_test.py +++ b/bindings/python/cntk/ops/tests/kernel_test.py @@ -69,6 +69,47 @@ def test_op_convolution_without_padding(convolution_map, convolution_input, devi unittest_helper(input_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision) + +ASYM_CONVOLUTION_DATA = [ + ([1, 1, 1, 3, 3], # input_size + [1, 2, 2], # convolution size + [[[[ 19, 25, 10], + [ 37, 43, 16], + [ 7, 8, 0]]]]) # result +] +# this test handles convolution with asymmetric padding, in particular, with auto_padding is set to True +# and the kernel shape is even +@pytest.mark.parametrize("input_size, conv_size, result", ASYM_CONVOLUTION_DATA) +def test_asym_convolution(input_size, conv_size, result, device_id, precision): + dt = PRECISION_TO_TYPE[precision] + dev = cntk_device(device_id) + + # fill input operand with a sequence 1,2,3,... til total size and then + # resize to input_size + total_size = np.prod(input_size) + x = np.arange(total_size, dtype=dt) + input_operand = x.reshape(input_size) + + a = I(shape=input_operand.shape[2:], + dtype=sanitize_dtype_cntk(precision), + needs_gradient=False, + name='a') + + # do the same for convolution kernel + total_size = np.prod(conv_size) + y = np.arange(total_size, dtype=dt) + conv_map = constant(value=y.reshape(conv_size), device=dev) + + from cntk import convolution + input_op = convolution(conv_map, a, auto_padding=[True]) + + forward_input = {a: input_operand} + expected_forward = AA([result]) + + unittest_helper(input_op, forward_input, expected_forward, + None, device_id=device_id, precision=precision) + + POOLING_GEOMETRY_DATA = [ ([1, 1, 1, 6, 6], # input_size (1, 5, 5), # pooling_window From ca6b17c377fd30e07983eac6e9a40f2f1567d01e Mon Sep 17 00:00:00 2001 From: Cha Zhang Date: Wed, 4 Jan 2017 07:49:51 -0800 Subject: [PATCH 011/120] Revert automatic format change by VS2013. --- Source/Math/CuDnnConvolutionEngine.cu | 40 ++++++++++++--------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu index 55f0d2f14fa7..6a339ad92e90 100644 --- a/Source/Math/CuDnnConvolutionEngine.cu +++ b/Source/Math/CuDnnConvolutionEngine.cu @@ -22,9 +22,7 @@ const char* CudaErrString(cudnnStatus_t x) #define TENSOR_FORMAT CUDNN_TENSOR_NCHW #define FILTER_FORMAT CUDNN_TENSOR_NCHW -namespace Microsoft { -namespace MSR { -namespace CNTK { +namespace Microsoft { namespace MSR { namespace CNTK { static bool IsGpu(DEVICEID_TYPE deviceId) { @@ -92,8 +90,8 @@ public: } SmallVector upscale(stride.size(), 1); CUDNN_CALL(cudnnSetConvolutionNdDescriptor(m_conv, (int)stride.size(), pad.data(), - stride.data(), upscale.data(), - CUDNN_CROSS_CORRELATION, dataType)); + stride.data(), upscale.data(), + CUDNN_CROSS_CORRELATION, dataType)); } ~CuDnnConv() @@ -141,9 +139,9 @@ public: // Must use CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING to get the same results as in reference engine. CUDNN_CALL(cudnnSetPoolingNdDescriptor(m_pool, - kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, - CUDNN_PROPAGATE_NAN, - (int)dims.size(), dims.data(), pad.data(), stride.data())); + kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, + CUDNN_PROPAGATE_NAN, + (int)dims.size(), dims.data(), pad.data(), stride.data())); } ~CuDnnPool() @@ -185,9 +183,7 @@ public: { } - virtual bool ImplementsGradientOverwriteOptimization() const override { - return true; - } + virtual bool ImplementsGradientOverwriteOptimization() const override { return true; } protected: using Base::m_geometry; @@ -208,8 +204,8 @@ protected: { if (m_kernelT == nullptr) { - m_kernelT = std::make_unique(*m_geometry, m_dataType), - m_conv = std::make_unique(*m_geometry, m_dataType); + m_kernelT = std::make_unique(*m_geometry, m_dataType), + m_conv = std::make_unique(*m_geometry, m_dataType); } } @@ -223,7 +219,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -271,7 +267,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -288,7 +284,7 @@ protected: workspace.Resize((m_backDataAlgo.Algo.memory + sizeof(ElemType) - 1) / sizeof(ElemType), 1); // Compute gradients with respect to the output tensor (data). CUDNN_CALL(cudnnConvolutionBackwardData(*m_cudnn, &C::One, *m_kernelT, ptr(kernel), m_outT, ptr(srcGrad), *m_conv, m_backDataAlgo.Algo.algo, - ptr(workspace), m_backDataAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, m_inT, ptr(grad))); + ptr(workspace), m_backDataAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, m_inT, ptr(grad))); workspace.Resize(0, 0); } @@ -302,7 +298,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -319,7 +315,7 @@ protected: workspace.Resize((m_backFiltAlgo.Algo.memory + sizeof(ElemType) - 1) / sizeof(ElemType), 1); // Compute gradients with respect to the output tensor (data). CUDNN_CALL(cudnnConvolutionBackwardFilter(*m_cudnn, &C::One, m_inT, ptr(in), m_outT, ptr(srcGrad), *m_conv, m_backFiltAlgo.Algo.algo, - ptr(workspace), m_backFiltAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, *m_kernelT, ptr(kernelGrad))); + ptr(workspace), m_backFiltAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, *m_kernelT, ptr(kernelGrad))); workspace.Resize(0, 0); } @@ -343,7 +339,7 @@ protected: m_inT.UpdateBatchSize(batchSize); m_outT.UpdateBatchSize(batchSize); CUDNN_CALL(cudnnPoolingBackward(*m_cudnn, *(m_pool), &C::One, m_outT, ptr(out), m_outT, ptr(srcGrad), - m_inT, ptr(in), &C::One, m_inT, ptr(grad))); + m_inT, ptr(in), &C::One, m_inT, ptr(grad))); } void MaxUnpoolingCore(const Mat& out, const Mat& poolIn, Mat& in) override @@ -397,7 +393,7 @@ private: size_t maxMem = m_maxTempMemSizeInSamples == 0 ? (std::numeric_limits::max)() : inputSampleSize * m_maxTempMemSizeInSamples * sizeof(ElemType); // Find best (fastest) algorithm which satisfies workspace requirements. auto res = std::find_if(algoPerf, algoPerf + calgo, - [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); + [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); if (res == algoPerf + calgo) RuntimeError("cuDNN could not find suitable algorithm for the current convolution configuration."); @@ -409,8 +405,8 @@ private: // Find fastest algorithm that does NOT require workspace. It is used as a fallback algo in Forward function. // Currently all Forward algorithms are deterministic, so no need for checking. - res = std::find_if(algoPerf, algoPerf + calgo, - [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); + res = std::find_if(algoPerf, algoPerf + calgo, + [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); if (res == algoPerf + calgo) { // In theory, this should never happen. From 1ed0f08dce1341a6e4c2657a985d6c5f08588b53 Mon Sep 17 00:00:00 2001 From: Cha Zhang Date: Tue, 3 Jan 2017 17:37:03 -0800 Subject: [PATCH 012/120] Bug fix for asymmetric padding for convolution. Added a test too. --- Source/Math/CuDnnConvolutionEngine.cu | 40 +++++++++++++++------------ 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu index 6a339ad92e90..55f0d2f14fa7 100644 --- a/Source/Math/CuDnnConvolutionEngine.cu +++ b/Source/Math/CuDnnConvolutionEngine.cu @@ -22,7 +22,9 @@ const char* CudaErrString(cudnnStatus_t x) #define TENSOR_FORMAT CUDNN_TENSOR_NCHW #define FILTER_FORMAT CUDNN_TENSOR_NCHW -namespace Microsoft { namespace MSR { namespace CNTK { +namespace Microsoft { +namespace MSR { +namespace CNTK { static bool IsGpu(DEVICEID_TYPE deviceId) { @@ -90,8 +92,8 @@ public: } SmallVector upscale(stride.size(), 1); CUDNN_CALL(cudnnSetConvolutionNdDescriptor(m_conv, (int)stride.size(), pad.data(), - stride.data(), upscale.data(), - CUDNN_CROSS_CORRELATION, dataType)); + stride.data(), upscale.data(), + CUDNN_CROSS_CORRELATION, dataType)); } ~CuDnnConv() @@ -139,9 +141,9 @@ public: // Must use CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING to get the same results as in reference engine. CUDNN_CALL(cudnnSetPoolingNdDescriptor(m_pool, - kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, - CUDNN_PROPAGATE_NAN, - (int)dims.size(), dims.data(), pad.data(), stride.data())); + kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, + CUDNN_PROPAGATE_NAN, + (int)dims.size(), dims.data(), pad.data(), stride.data())); } ~CuDnnPool() @@ -183,7 +185,9 @@ public: { } - virtual bool ImplementsGradientOverwriteOptimization() const override { return true; } + virtual bool ImplementsGradientOverwriteOptimization() const override { + return true; + } protected: using Base::m_geometry; @@ -204,8 +208,8 @@ protected: { if (m_kernelT == nullptr) { - m_kernelT = std::make_unique(*m_geometry, m_dataType), - m_conv = std::make_unique(*m_geometry, m_dataType); + m_kernelT = std::make_unique(*m_geometry, m_dataType), + m_conv = std::make_unique(*m_geometry, m_dataType); } } @@ -219,7 +223,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -267,7 +271,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -284,7 +288,7 @@ protected: workspace.Resize((m_backDataAlgo.Algo.memory + sizeof(ElemType) - 1) / sizeof(ElemType), 1); // Compute gradients with respect to the output tensor (data). CUDNN_CALL(cudnnConvolutionBackwardData(*m_cudnn, &C::One, *m_kernelT, ptr(kernel), m_outT, ptr(srcGrad), *m_conv, m_backDataAlgo.Algo.algo, - ptr(workspace), m_backDataAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, m_inT, ptr(grad))); + ptr(workspace), m_backDataAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, m_inT, ptr(grad))); workspace.Resize(0, 0); } @@ -298,7 +302,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -315,7 +319,7 @@ protected: workspace.Resize((m_backFiltAlgo.Algo.memory + sizeof(ElemType) - 1) / sizeof(ElemType), 1); // Compute gradients with respect to the output tensor (data). CUDNN_CALL(cudnnConvolutionBackwardFilter(*m_cudnn, &C::One, m_inT, ptr(in), m_outT, ptr(srcGrad), *m_conv, m_backFiltAlgo.Algo.algo, - ptr(workspace), m_backFiltAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, *m_kernelT, ptr(kernelGrad))); + ptr(workspace), m_backFiltAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, *m_kernelT, ptr(kernelGrad))); workspace.Resize(0, 0); } @@ -339,7 +343,7 @@ protected: m_inT.UpdateBatchSize(batchSize); m_outT.UpdateBatchSize(batchSize); CUDNN_CALL(cudnnPoolingBackward(*m_cudnn, *(m_pool), &C::One, m_outT, ptr(out), m_outT, ptr(srcGrad), - m_inT, ptr(in), &C::One, m_inT, ptr(grad))); + m_inT, ptr(in), &C::One, m_inT, ptr(grad))); } void MaxUnpoolingCore(const Mat& out, const Mat& poolIn, Mat& in) override @@ -393,7 +397,7 @@ private: size_t maxMem = m_maxTempMemSizeInSamples == 0 ? (std::numeric_limits::max)() : inputSampleSize * m_maxTempMemSizeInSamples * sizeof(ElemType); // Find best (fastest) algorithm which satisfies workspace requirements. auto res = std::find_if(algoPerf, algoPerf + calgo, - [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); + [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); if (res == algoPerf + calgo) RuntimeError("cuDNN could not find suitable algorithm for the current convolution configuration."); @@ -405,8 +409,8 @@ private: // Find fastest algorithm that does NOT require workspace. It is used as a fallback algo in Forward function. // Currently all Forward algorithms are deterministic, so no need for checking. - res = std::find_if(algoPerf, algoPerf + calgo, - [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); + res = std::find_if(algoPerf, algoPerf + calgo, + [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); if (res == algoPerf + calgo) { // In theory, this should never happen. From a0ad56ba756ab471e8298dcea0beb3f7b13070a3 Mon Sep 17 00:00:00 2001 From: Cha Zhang Date: Wed, 4 Jan 2017 07:49:51 -0800 Subject: [PATCH 013/120] Revert automatic format change by VS2013. --- Source/Math/CuDnnConvolutionEngine.cu | 40 ++++++++++++--------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu index 55f0d2f14fa7..6a339ad92e90 100644 --- a/Source/Math/CuDnnConvolutionEngine.cu +++ b/Source/Math/CuDnnConvolutionEngine.cu @@ -22,9 +22,7 @@ const char* CudaErrString(cudnnStatus_t x) #define TENSOR_FORMAT CUDNN_TENSOR_NCHW #define FILTER_FORMAT CUDNN_TENSOR_NCHW -namespace Microsoft { -namespace MSR { -namespace CNTK { +namespace Microsoft { namespace MSR { namespace CNTK { static bool IsGpu(DEVICEID_TYPE deviceId) { @@ -92,8 +90,8 @@ public: } SmallVector upscale(stride.size(), 1); CUDNN_CALL(cudnnSetConvolutionNdDescriptor(m_conv, (int)stride.size(), pad.data(), - stride.data(), upscale.data(), - CUDNN_CROSS_CORRELATION, dataType)); + stride.data(), upscale.data(), + CUDNN_CROSS_CORRELATION, dataType)); } ~CuDnnConv() @@ -141,9 +139,9 @@ public: // Must use CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING to get the same results as in reference engine. CUDNN_CALL(cudnnSetPoolingNdDescriptor(m_pool, - kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, - CUDNN_PROPAGATE_NAN, - (int)dims.size(), dims.data(), pad.data(), stride.data())); + kind == PoolKind::Max && !forceDeterministicAlgorithms ? CUDNN_POOLING_MAX : CUDNN_POOLING_AVERAGE_COUNT_EXCLUDE_PADDING, + CUDNN_PROPAGATE_NAN, + (int)dims.size(), dims.data(), pad.data(), stride.data())); } ~CuDnnPool() @@ -185,9 +183,7 @@ public: { } - virtual bool ImplementsGradientOverwriteOptimization() const override { - return true; - } + virtual bool ImplementsGradientOverwriteOptimization() const override { return true; } protected: using Base::m_geometry; @@ -208,8 +204,8 @@ protected: { if (m_kernelT == nullptr) { - m_kernelT = std::make_unique(*m_geometry, m_dataType), - m_conv = std::make_unique(*m_geometry, m_dataType); + m_kernelT = std::make_unique(*m_geometry, m_dataType), + m_conv = std::make_unique(*m_geometry, m_dataType); } } @@ -223,7 +219,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionFwdAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -271,7 +267,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionBwdDataAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_DATA_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -288,7 +284,7 @@ protected: workspace.Resize((m_backDataAlgo.Algo.memory + sizeof(ElemType) - 1) / sizeof(ElemType), 1); // Compute gradients with respect to the output tensor (data). CUDNN_CALL(cudnnConvolutionBackwardData(*m_cudnn, &C::One, *m_kernelT, ptr(kernel), m_outT, ptr(srcGrad), *m_conv, m_backDataAlgo.Algo.algo, - ptr(workspace), m_backDataAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, m_inT, ptr(grad))); + ptr(workspace), m_backDataAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, m_inT, ptr(grad))); workspace.Resize(0, 0); } @@ -302,7 +298,7 @@ protected: if (m_forceDeterministicAlgorithms) { auto found = std::find_if(algoPerf, algoPerf + calgo, - [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); + [](const cudnnConvolutionBwdFilterAlgoPerf_t& a) { return a.algo == CUDNN_CONVOLUTION_BWD_FILTER_ALGO_1 && a.status == CUDNN_STATUS_SUCCESS; }); if (found == algoPerf + calgo) RuntimeError("cuDNN could not find a deterministic algorithm. Set 'forceDeterministicAlgorithms=false' in your configuration."); calgo = 1; @@ -319,7 +315,7 @@ protected: workspace.Resize((m_backFiltAlgo.Algo.memory + sizeof(ElemType) - 1) / sizeof(ElemType), 1); // Compute gradients with respect to the output tensor (data). CUDNN_CALL(cudnnConvolutionBackwardFilter(*m_cudnn, &C::One, m_inT, ptr(in), m_outT, ptr(srcGrad), *m_conv, m_backFiltAlgo.Algo.algo, - ptr(workspace), m_backFiltAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, *m_kernelT, ptr(kernelGrad))); + ptr(workspace), m_backFiltAlgo.Algo.memory, accumulateGradient ? &C::One : &C::Zero, *m_kernelT, ptr(kernelGrad))); workspace.Resize(0, 0); } @@ -343,7 +339,7 @@ protected: m_inT.UpdateBatchSize(batchSize); m_outT.UpdateBatchSize(batchSize); CUDNN_CALL(cudnnPoolingBackward(*m_cudnn, *(m_pool), &C::One, m_outT, ptr(out), m_outT, ptr(srcGrad), - m_inT, ptr(in), &C::One, m_inT, ptr(grad))); + m_inT, ptr(in), &C::One, m_inT, ptr(grad))); } void MaxUnpoolingCore(const Mat& out, const Mat& poolIn, Mat& in) override @@ -397,7 +393,7 @@ private: size_t maxMem = m_maxTempMemSizeInSamples == 0 ? (std::numeric_limits::max)() : inputSampleSize * m_maxTempMemSizeInSamples * sizeof(ElemType); // Find best (fastest) algorithm which satisfies workspace requirements. auto res = std::find_if(algoPerf, algoPerf + calgo, - [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); + [=](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory <= maxMem; }); if (res == algoPerf + calgo) RuntimeError("cuDNN could not find suitable algorithm for the current convolution configuration."); @@ -409,8 +405,8 @@ private: // Find fastest algorithm that does NOT require workspace. It is used as a fallback algo in Forward function. // Currently all Forward algorithms are deterministic, so no need for checking. - res = std::find_if(algoPerf, algoPerf + calgo, - [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); + res = std::find_if(algoPerf, algoPerf + calgo, + [](const CuDnnAlgoT& cur) { return cur.status == CUDNN_STATUS_SUCCESS && cur.memory == 0; }); if (res == algoPerf + calgo) { // In theory, this should never happen. From 02291c55b4b868236a57dc3b6b230001bf55e498 Mon Sep 17 00:00:00 2001 From: Cha Zhang Date: Wed, 4 Jan 2017 10:09:05 -0800 Subject: [PATCH 014/120] Bug fix when lowerPad and upperPad has lower ranks than kernelPad. --- Source/Math/CuDnnConvolutionEngine.cu | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu index 6a339ad92e90..513917937220 100644 --- a/Source/Math/CuDnnConvolutionEngine.cu +++ b/Source/Math/CuDnnConvolutionEngine.cu @@ -514,16 +514,18 @@ bool CuDnnConvolutionEngineFactory::IsSupported(DEVICEID_TYPE deviceId // detect asymmetric padding (either by auto-padding or by specifying lowerPad/upperPad) and choose // the reference convolution implementation instead const auto& autopad = geometry->AutoPad(); + assert (autopad.size() == kernelRank); const auto& lowerpad = geometry->LowerPad(); - const auto& upperpad = geometry->UpperPad(); + const auto& upperpad = geometry->UpperPad(); + const auto& padRank = min(lowerpad.GetRank(), upperpad.GetRank()); // there are cases where padRank < kernelRank for (int i = 0; i < kernelRank; i++) { if (autopad[i]) retVal = retVal && (kernel[i] % 2 != 0); // make sure kernel size is odd - else + else if (i < padRank) retVal = retVal && (lowerpad[i] == upperpad[i]); // make sure lower and upper padding are equal } - return retVal; + return retVal; } template class CuDnnConvolutionEngineFactory; From cfd6083a019550704ea6c86e1b10ce3575c2a3b8 Mon Sep 17 00:00:00 2001 From: Cha Zhang Date: Wed, 4 Jan 2017 15:19:38 -0800 Subject: [PATCH 015/120] Fix autoPad issue using GetAutoPad(). --- Source/Math/ConvolveGeometry.h | 19 +++++++++++++++++++ Source/Math/CuDnnConvolutionEngine.cu | 23 ++++++++++------------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/Source/Math/ConvolveGeometry.h b/Source/Math/ConvolveGeometry.h index 95ca288d3ae5..480aed82458d 100644 --- a/Source/Math/ConvolveGeometry.h +++ b/Source/Math/ConvolveGeometry.h @@ -390,6 +390,25 @@ class ConvolveGeometry final return -(center - (kernSize - 1) / 2); } + int GetUpperPad(size_t dim) const + { + if (!GetAutoPad(dim)) + return (int)m_upperPad[m_upperPad.size() == 1 ? 0 : dim]; + + int kernSize = (int)m_kernelShape[dim]; + int inpSize = (int)m_inputShape[dim]; + int outSize = (int)m_outputShape[dim]; + int stride = (int)GetStride(dim); + + // Taken from computation in ConvolveGeometry ctor. + // Number of cells between first and last "centers", inclusive. + int cells = (outSize - 1) * stride + 1; + // Extra cells, to the left and right of "cells". + int extra = inpSize - cells; + int center = extra / 2; + return (kernSize - 1) - (kernSize - 1) / 2 - (extra - center); + } + // Computes output shape given input shape and other convolution parameters. static TensorShape ComputeOutputShape(const TensorShape& inputShape, const TensorShape& kernelShape, const TensorShape& mapCount, const TensorShape& stride, const BoolVec& sharing, const BoolVec& autoPad, const TensorShape& lowerPad, const TensorShape& upperPad) diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu index 513917937220..4852183ee18c 100644 --- a/Source/Math/CuDnnConvolutionEngine.cu +++ b/Source/Math/CuDnnConvolutionEngine.cu @@ -510,20 +510,17 @@ bool CuDnnConvolutionEngineFactory::IsSupported(DEVICEID_TYPE deviceId (poolKind == PoolKind::None || inputRank <= 3 && (kernelRank < 3 || kernel[2] == 1))); - // cuDNN as of version 8.0 does not handle asymmetric padding correctly. We need to - // detect asymmetric padding (either by auto-padding or by specifying lowerPad/upperPad) and choose - // the reference convolution implementation instead - const auto& autopad = geometry->AutoPad(); - assert (autopad.size() == kernelRank); - const auto& lowerpad = geometry->LowerPad(); - const auto& upperpad = geometry->UpperPad(); - const auto& padRank = min(lowerpad.GetRank(), upperpad.GetRank()); // there are cases where padRank < kernelRank - for (int i = 0; i < kernelRank; i++) + // cuDNN as of version 8.0 does not handle asymmetric padding for convolution correctly. We need to detect asymmetric + // padding due to auto-padding and choose the reference convolution implementation instead + if (poolKind == PoolKind::None) // only for convolution, pooling seems fine { - if (autopad[i]) - retVal = retVal && (kernel[i] % 2 != 0); // make sure kernel size is odd - else if (i < padRank) - retVal = retVal && (lowerpad[i] == upperpad[i]); // make sure lower and upper padding are equal + for (int i = 0; i < kernelRank; i++) + { + if (geometry->GetAutoPad(i)) + retVal = retVal && (kernel[i] % 2 != 0); // make sure kernel size is odd + else + retVal = retVal && (geometry->GetLowerPad(i) == geometry->GetUpperPad(i)); // lower pad is same as upper pad + } } return retVal; } From 59dcc28af5dfc8e8b533e9dcbc2d49daa44a4fa4 Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Wed, 4 Jan 2017 16:39:50 -0800 Subject: [PATCH 016/120] Some more self-assignment unit tests just in case --- Tests/UnitTests/MathTests/MatrixTests.cpp | 33 +++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/Tests/UnitTests/MathTests/MatrixTests.cpp b/Tests/UnitTests/MathTests/MatrixTests.cpp index fba0ea48fa1e..eab7e41bd1d4 100644 --- a/Tests/UnitTests/MathTests/MatrixTests.cpp +++ b/Tests/UnitTests/MathTests/MatrixTests.cpp @@ -793,6 +793,39 @@ BOOST_FIXTURE_TEST_CASE(MatrixAssignXOf, RandomSeedFixture) b.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + // Check for self-assignment 3 + c.AssignSumOf(a, b); + c.AddElementProductOf(a, c); + foreach_coord(i, j, c) + { + BOOST_CHECK_CLOSE(c(i, j), (1 + a(i, j)) * (a(i, j) + b(i, j)), tolerance); + } + a.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + b.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + + // Check for self-assignment 4 + c.AssignSumOf(a, b); + c.AddElementProductOf(c, a); + foreach_coord(i, j, c) + { + BOOST_CHECK_CLOSE(c(i, j), (1 + a(i, j)) * (a(i, j) + b(i, j)), tolerance); + } + a.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + b.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + + // Check for self-assignment 5 + c.AssignSumOf(a, b); + c.AddElementProductOf(c, c); + foreach_coord(i, j, c) + { + BOOST_CHECK_CLOSE(c(i, j), (1 + a(i, j) + b(i, j)) * (a(i, j) + b(i, j)), tolerance); + } + a.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + b.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); + // AssignElementProductOf c.AssignElementProductOf(a, b); foreach_coord (i, j, c) From ada156e48bbd2b8396715c71385742b67c93032c Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 5 Jan 2017 08:11:46 +0000 Subject: [PATCH 017/120] configure: don't pick up Python versions implicitly --- configure | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/configure b/configure index 9704c24d2421..60afd72ef658 100755 --- a/configure +++ b/configure @@ -904,10 +904,12 @@ fi if test $enable_python = yes then + # TODO disabled for now. Default Python versions may miss dependencies we do not + # check for here yet, only support explicitly specified paths. # Fill unfilled default paths - [ -z "${py_paths[27]}" ] && find_python 27 && py_paths[27]=$(find_python 27) - [ -z "${py_paths[34]}" ] && find_python 34 && py_paths[34]=$(find_python 34) - [ -z "${py_paths[35]}" ] && find_python 35 && py_paths[35]=$(find_python 35) + #[ -z "${py_paths[27]}" ] && find_python 27 && py_paths[27]=$(find_python 27) + #[ -z "${py_paths[34]}" ] && find_python 34 && py_paths[34]=$(find_python 34) + #[ -z "${py_paths[35]}" ] && find_python 35 && py_paths[35]=$(find_python 35) # Unless there's a specified order or restriction, take all configured Python versions (old to new) [ -z "$py_versions" ] && py_versions=${!py_paths[@]} From 8aa380bd2440bc7d7a82f75eb44f9e3838ab67ab Mon Sep 17 00:00:00 2001 From: Eldar Akchurin Date: Tue, 3 Jan 2017 18:09:07 +0100 Subject: [PATCH 018/120] Allowing deserializers return invalid sequences --- .../CompositeDataReader.cpp | 7 +- .../ImageReader/Base64ImageDeserializer.cpp | 28 +- .../ImageReader/ImageDeserializerBase.cpp | 31 +- Source/Readers/ReaderLib/BlockRandomizer.cpp | 8 +- Source/Readers/ReaderLib/BlockRandomizer.h | 9 +- Source/Readers/ReaderLib/DataDeserializer.h | 3 +- Source/Readers/ReaderLib/Indexer.cpp | 2 +- Source/Readers/ReaderLib/NoRandomizer.cpp | 6 +- Source/Readers/ReaderLib/NoRandomizer.h | 11 +- Source/Readers/ReaderLib/ReaderUtil.h | 73 +- Source/Readers/ReaderLib/StringToIdMap.h | 2 +- .../UnitTests/ReaderTests/baseline.txt | 1011 +++++++++-------- .../InvalidBase64ImageReaderSimple_map.txt | 1 + .../ReaderTests/ImageReaderTests.cpp | 33 +- 14 files changed, 678 insertions(+), 547 deletions(-) create mode 100644 Tests/UnitTests/ReaderTests/Data/InvalidBase64ImageReaderSimple_map.txt diff --git a/Source/Readers/CompositeDataReader/CompositeDataReader.cpp b/Source/Readers/CompositeDataReader/CompositeDataReader.cpp index 4f8704487060..0f90c19a4fd1 100644 --- a/Source/Readers/CompositeDataReader/CompositeDataReader.cpp +++ b/Source/Readers/CompositeDataReader/CompositeDataReader.cpp @@ -96,6 +96,9 @@ CompositeDataReader::CompositeDataReader(const ConfigParameters& config) : // Pick up the randomizer, always picking up no randomization for the write mode. bool randomize = isActionWrite ? false : config(L"randomize", true); + // Get maximum number of allowed errors per worker. + size_t maxErrors = config(L"maxErrors", 0); + // By default do not use omp threads for deserialization of sequences. // It makes sense to put it to true for cases when deserialization is CPU intensive, // i.e. decompression of images. @@ -115,11 +118,11 @@ CompositeDataReader::CompositeDataReader(const ConfigParameters& config) : randomizationWindow = config(L"randomizationWindow", randomizationWindow); bool shouldPrefetch = true; - m_sequenceEnumerator = std::make_shared(verbosity, randomizationWindow, deserializer, shouldPrefetch, multiThreadedDeserialization); + m_sequenceEnumerator = std::make_shared(verbosity, randomizationWindow, deserializer, shouldPrefetch, multiThreadedDeserialization, maxErrors); } else { - m_sequenceEnumerator = std::make_shared(deserializer, multiThreadedDeserialization); + m_sequenceEnumerator = std::make_shared(deserializer, multiThreadedDeserialization, maxErrors); } // In case when there are transforms, applying them to the data. diff --git a/Source/Readers/ImageReader/Base64ImageDeserializer.cpp b/Source/Readers/ImageReader/Base64ImageDeserializer.cpp index 9a470bb3e0bb..72e1c1bb42bb 100644 --- a/Source/Readers/ImageReader/Base64ImageDeserializer.cpp +++ b/Source/Readers/ImageReader/Base64ImageDeserializer.cpp @@ -41,7 +41,7 @@ namespace Microsoft { namespace MSR { namespace CNTK { // Read chunk into memory. int rc = _fseeki64(m_deserializer.m_dataFile.get(), m_chunkOffset, SEEK_SET); if (rc) - RuntimeError("Error seeking to position %" PRId64 " in the input file (%ls), error %d", m_chunkOffset, m_deserializer.m_fileName.c_str(), rc); + RuntimeError("Error seeking to position '%" PRId64 "' in the input file '%ls', error code '%d'", m_chunkOffset, m_deserializer.m_fileName.c_str(), rc); freadOrDie(m_buffer.data(), descriptor.m_byteSize, 1, m_deserializer.m_dataFile.get()); } @@ -69,13 +69,13 @@ namespace Microsoft { namespace MSR { namespace CNTK { // Let's get the label. if (!token) - RuntimeError("Empty label value for sequence '%s'", KeyOf(sequence).c_str()); + RuntimeError("Empty label value for sequence '%s' in the input file '%ls'", KeyOf(sequence).c_str(), m_deserializer.m_fileName.c_str()); char* eptr = nullptr; errno = 0; size_t classId = strtoull(token, &eptr, 10); if (token == eptr || errno == ERANGE) - RuntimeError("Cannot parse label value for sequence '%s'", KeyOf(sequence).c_str()); + RuntimeError("Cannot parse label value for sequence '%s' in the input file '%ls'", KeyOf(sequence).c_str(), m_deserializer.m_fileName.c_str()); size_t labelDimension = m_deserializer.m_labelGenerator->LabelDimension(); if (classId >= labelDimension) @@ -91,22 +91,24 @@ namespace Microsoft { namespace MSR { namespace CNTK { // Find line end or end of buffer. char* endToken = strchr(token, 0); if (!endToken) - RuntimeError("Cannot find the end of the image for sequence '%s'", KeyOf(sequence).c_str()); + RuntimeError("Cannot find the end of the image for sequence '%s' in the input file '%ls'", KeyOf(sequence).c_str(), m_deserializer.m_fileName.c_str()); // Remove non base64 characters at the end of the string (tabs/spaces) while (endToken > token && !IsBase64Char(*(endToken - 1))) endToken--; - std::vector decodedImage = DecodeBase64(token, endToken); - cv::Mat img = cv::imdecode(decodedImage, m_deserializer.m_grayscale ? cv::IMREAD_GRAYSCALE : cv::IMREAD_COLOR); - - auto image = std::make_shared(); - image->m_image = std::move(img); - auto& cvImage = image->m_image; - if (!cvImage.data) - RuntimeError("Cannot decode sequence '%s'", KeyOf(sequence).c_str()); + std::vector decodedImage; + cv::Mat image; + if (!DecodeBase64(token, endToken, decodedImage)) + { + fprintf(stderr, "WARNING: Cannot decode sequence with id %" PRIu64 " in the input file '%ls'\n", sequence.m_key.m_sequence, m_deserializer.m_fileName.c_str()); + } + else + { + image = cv::imdecode(decodedImage, m_deserializer.m_grayscale ? cv::IMREAD_GRAYSCALE : cv::IMREAD_COLOR); + } - m_deserializer.PopulateSequenceData(cvImage, classId, sequenceId, result); + m_deserializer.PopulateSequenceData(image, classId, sequenceId, result); } }; diff --git a/Source/Readers/ImageReader/ImageDeserializerBase.cpp b/Source/Readers/ImageReader/ImageDeserializerBase.cpp index ff6f711e4e5c..feafa34bb04f 100644 --- a/Source/Readers/ImageReader/ImageDeserializerBase.cpp +++ b/Source/Readers/ImageReader/ImageDeserializerBase.cpp @@ -71,18 +71,27 @@ namespace Microsoft { namespace MSR { namespace CNTK { void ImageDeserializerBase::PopulateSequenceData(cv::Mat image, size_t classId, size_t sequenceId, std::vector& result) { - ElementType dataType = ConvertImageToSupportedDataType(image, m_precision); - if (!image.isContinuous()) - image = image.clone(); - assert(image.isContinuous()); - - ImageDimensions dimensions(image.cols, image.rows, image.channels()); auto imageData = make_shared(); - imageData->m_sampleLayout = std::make_shared(dimensions.AsTensorShape(HWC)); - imageData->m_id = sequenceId; - imageData->m_image = image; - imageData->m_numberOfSamples = 1; - imageData->m_elementType = dataType; + if (!image.data) + { + imageData->m_isValid = false; + } + else + { + ElementType dataType = ConvertImageToSupportedDataType(image, m_precision); + if (!image.isContinuous()) + image = image.clone(); + assert(image.isContinuous()); + + ImageDimensions dimensions(image.cols, image.rows, image.channels()); + + imageData->m_sampleLayout = std::make_shared(dimensions.AsTensorShape(HWC)); + imageData->m_id = sequenceId; + imageData->m_image = image; + imageData->m_numberOfSamples = 1; + imageData->m_elementType = dataType; + imageData->m_isValid = true; + } result.push_back(imageData); auto label = std::make_shared(); diff --git a/Source/Readers/ReaderLib/BlockRandomizer.cpp b/Source/Readers/ReaderLib/BlockRandomizer.cpp index bcd60333dcb6..43b3da4b1ffa 100644 --- a/Source/Readers/ReaderLib/BlockRandomizer.cpp +++ b/Source/Readers/ReaderLib/BlockRandomizer.cpp @@ -21,7 +21,8 @@ BlockRandomizer::BlockRandomizer( size_t randomizationRangeInSamples, IDataDeserializerPtr deserializer, bool shouldPrefetch, - bool multithreadedGetNextSequence) + bool multithreadedGetNextSequence, + size_t maxNumberOfInvalidSequences) : m_verbosity(verbosity), m_deserializer(deserializer), m_sweep(SIZE_MAX), @@ -31,7 +32,8 @@ BlockRandomizer::BlockRandomizer( m_sweepTotalNumberOfSamples(0), m_chunkRandomizer(std::make_shared(deserializer, randomizationRangeInSamples)), m_multithreadedGetNextSequences(multithreadedGetNextSequence), - m_prefetchedChunk(CHUNKID_MAX) + m_prefetchedChunk(CHUNKID_MAX), + m_cleaner(maxNumberOfInvalidSequences) { assert(deserializer != nullptr); @@ -156,6 +158,8 @@ Sequences BlockRandomizer::GetNextSequences(size_t globalSampleCount, size_t loc process(i); } + m_cleaner.Clean(result); + // Now it is safe to start the new chunk prefetch. ChunkIdType chunkToPrefetchNext = GetChunkToPrefetch(windowRange); Prefetch(chunkToPrefetchNext); diff --git a/Source/Readers/ReaderLib/BlockRandomizer.h b/Source/Readers/ReaderLib/BlockRandomizer.h index 8da7c6dadae7..23d9024787d7 100644 --- a/Source/Readers/ReaderLib/BlockRandomizer.h +++ b/Source/Readers/ReaderLib/BlockRandomizer.h @@ -11,6 +11,7 @@ #include "DataDeserializer.h" #include "ChunkRandomizer.h" #include "SequenceRandomizer.h" +#include "ReaderUtil.h" #include namespace Microsoft { namespace MSR { namespace CNTK { @@ -40,7 +41,8 @@ class BlockRandomizer : public SequenceEnumerator size_t randomizationRangeInSamples, IDataDeserializerPtr deserializer, bool shouldPrefetch, - bool multithreadedGetNextSequences = false); + bool multithreadedGetNextSequences = false, + size_t maxNumberOfInvalidSequences = 0); // per worker // Starts a new epoch. virtual void StartEpoch(const EpochConfiguration& config) override; @@ -147,9 +149,8 @@ class BlockRandomizer : public SequenceEnumerator // Sequence buffer, used to avoid reallocation only. std::vector m_sequenceBuffer; - // Flag indicating whether in distributed mode the minibatch is filled completely - // with data local to the worker. - bool m_useLocalTimeline; + // Helper class for removing invalid sequences. + SequenceCleaner m_cleaner; }; }}} diff --git a/Source/Readers/ReaderLib/DataDeserializer.h b/Source/Readers/ReaderLib/DataDeserializer.h index 7b94531cabdb..f720d7d1bc5f 100644 --- a/Source/Readers/ReaderLib/DataDeserializer.h +++ b/Source/Readers/ReaderLib/DataDeserializer.h @@ -60,7 +60,7 @@ typedef std::shared_ptr SequenceDescriptionPtr; // TODO: add type casts (As() or AsRef<>() or AsPtr<>()) to subclasses as members here. struct SequenceDataBase { - SequenceDataBase() : m_id(0), m_numberOfSamples(0), m_elementType(ElementType::tvariant) {} + SequenceDataBase() : m_id(0), m_numberOfSamples(0), m_elementType(ElementType::tvariant), m_isValid(true) {} virtual ~SequenceDataBase() = default; // Sequence id. @@ -73,6 +73,7 @@ struct SequenceDataBase ElementType m_elementType; // Sequence element type. TensorShapePtr m_sampleLayout; // Sample layout, can be shared by several sequences. + bool m_isValid; // Flag indicating if sequence is valid. }; typedef std::shared_ptr SequenceDataPtr; diff --git a/Source/Readers/ReaderLib/Indexer.cpp b/Source/Readers/ReaderLib/Indexer.cpp index 9364589d68ff..83da03f17364 100644 --- a/Source/Readers/ReaderLib/Indexer.cpp +++ b/Source/Readers/ReaderLib/Indexer.cpp @@ -166,7 +166,7 @@ void Indexer::AddSequenceIfIncluded(CorpusDescriptorPtr corpus, size_t sequenceI auto key = std::to_string(sequenceId); if (corpus->IsIncluded(key)) { - sd.m_key.m_sequence = sequenceId; + sd.m_key.m_sequence = corpus->KeyToId(key); sd.m_key.m_sample = 0; m_index.AddSequence(sd); } diff --git a/Source/Readers/ReaderLib/NoRandomizer.cpp b/Source/Readers/ReaderLib/NoRandomizer.cpp index 07ad282efb44..698cea301596 100644 --- a/Source/Readers/ReaderLib/NoRandomizer.cpp +++ b/Source/Readers/ReaderLib/NoRandomizer.cpp @@ -12,14 +12,15 @@ namespace Microsoft { namespace MSR { namespace CNTK { -NoRandomizer::NoRandomizer(IDataDeserializerPtr deserializer, bool multithreadedGetNextSequences) + NoRandomizer::NoRandomizer(IDataDeserializerPtr deserializer, bool multithreadedGetNextSequences, size_t maxNumberOfInvalidSequences) : m_deserializer(deserializer), m_currentChunkPosition(CHUNKID_MAX), m_globalSamplePosition(0), m_globalSequencePosition(0), m_totalNumberOfSamples(0), m_currentSequencePositionInChunk(0), - m_multithreadedGetNextSequences(multithreadedGetNextSequences) + m_multithreadedGetNextSequences(multithreadedGetNextSequences), + m_cleaner(maxNumberOfInvalidSequences) { assert(deserializer != nullptr); m_streams = m_deserializer->GetStreamDescriptions(); @@ -220,6 +221,7 @@ Sequences NoRandomizer::GetNextSequences(size_t globalSampleCount, size_t localS process(i); } + m_cleaner.Clean(result); return result; } diff --git a/Source/Readers/ReaderLib/NoRandomizer.h b/Source/Readers/ReaderLib/NoRandomizer.h index 7bc0c7ff9723..7e39e057f356 100644 --- a/Source/Readers/ReaderLib/NoRandomizer.h +++ b/Source/Readers/ReaderLib/NoRandomizer.h @@ -8,6 +8,7 @@ #include #include "SequenceEnumerator.h" #include "DataDeserializer.h" +#include "ReaderUtil.h" namespace Microsoft { namespace MSR { namespace CNTK { @@ -18,7 +19,10 @@ namespace Microsoft { namespace MSR { namespace CNTK { class NoRandomizer : public SequenceEnumerator { public: - NoRandomizer(IDataDeserializerPtr deserializer, bool multithreadedGetNextSequences = false); + NoRandomizer( + IDataDeserializerPtr deserializer, + bool multithreadedGetNextSequences = false, + size_t maxNumberOfInvalidSequences = 0); // per worker virtual void StartEpoch(const EpochConfiguration& config) override; virtual Sequences GetNextSequences(size_t globalSampleCount, size_t localSampleCount) override; @@ -89,9 +93,8 @@ class NoRandomizer : public SequenceEnumerator // Temp buffer to avoid allocations. std::vector m_sequenceBuffer; - // Flag, indicating whether in distributed mode the minibatch is filled completely - // with data local to the worker. - bool m_useLocalTimeline; + // Helper class for removing invalid sequences. + SequenceCleaner m_cleaner; }; }}} diff --git a/Source/Readers/ReaderLib/ReaderUtil.h b/Source/Readers/ReaderLib/ReaderUtil.h index a9df525c1c8d..71629dbbb03d 100644 --- a/Source/Readers/ReaderLib/ReaderUtil.h +++ b/Source/Readers/ReaderLib/ReaderUtil.h @@ -6,6 +6,7 @@ #pragma once #include "Reader.h" +#include "SequenceEnumerator.h" namespace Microsoft { namespace MSR { namespace CNTK { @@ -53,14 +54,14 @@ inline bool IsBase64Char(char c) return isalnum(c) || c == '/' || c == '+' || c == '='; } -inline std::vector DecodeBase64(const char* begin, const char* end) +inline bool DecodeBase64(const char* begin, const char* end, std::vector& result) { assert(std::find_if(begin, end, [](char c) { return !IsBase64Char(c); }) == end); size_t length = end - begin; if (length % 4 != 0) - RuntimeError("Invalid base64 data, length '%d' is not divisible by 4.", (int)length); - std::vector result; + return false; + result.resize((length * 3) / 4); // Upper bound on the max number of decoded symbols. size_t currentDecodedIndex = 0; while (begin < end) @@ -74,7 +75,71 @@ inline std::vector DecodeBase64(const char* begin, const char* end) // In Base 64 each 3 characters are encoded with 4 bytes. Plus there could be padding (last two bytes) size_t resultingLength = (length * 3) / 4 - (*(end - 2) == '=' ? 2 : (*(end - 1) == '=' ? 1 : 0)); result.resize(resultingLength); - return result; + return true; } +// Class to clean/keep track of invalid sequences. +class SequenceCleaner +{ +public: + SequenceCleaner(size_t maxNumberOfInvalidSequences) + : m_maxNumberOfInvalidSequences(maxNumberOfInvalidSequences), + m_numberOfCleanedSequences(0) + {} + + // Removes invalid sequences in place. + void Clean(Sequences& sequences) + { + if (sequences.m_data.empty()) + return; + + size_t clean = 0; + for (size_t i = 0; i < sequences.m_data.front().size(); ++i) + { + bool invalid = false; + for (const auto& s : sequences.m_data) + { + if (!s[i]->m_isValid) + { + invalid = true; + break; + } + } + + if (invalid) + { + m_numberOfCleanedSequences++; + continue; + } + + // For all streams reassign the sequence. + for (auto& s : sequences.m_data) + s[clean] = s[i]; + clean++; + } + + if (clean == 0) + { + sequences.m_data.resize(0); + return; + } + + // For all streams set new size. + for (auto& s : sequences.m_data) + s.resize(clean); + + if (m_numberOfCleanedSequences > m_maxNumberOfInvalidSequences) + RuntimeError("Number of invalid sequences '%d' in the input exceeded the specified maximum number '%d'", + (int)m_numberOfCleanedSequences, + (int)m_maxNumberOfInvalidSequences); + } + +private: + // Number of sequences cleaned. + size_t m_numberOfCleanedSequences; + + // Max number of allowed invalid sequences. + size_t m_maxNumberOfInvalidSequences; +}; + }}} diff --git a/Source/Readers/ReaderLib/StringToIdMap.h b/Source/Readers/ReaderLib/StringToIdMap.h index 045c434fff10..7e037f495bf4 100644 --- a/Source/Readers/ReaderLib/StringToIdMap.h +++ b/Source/Readers/ReaderLib/StringToIdMap.h @@ -69,7 +69,7 @@ class TStringToIdMap // Get string value by its integer id. const TString& operator[](size_t id) const { - if (id < m_indexedValues.size()) + if (id >= m_indexedValues.size()) RuntimeError("Unknown id requested"); return *m_indexedValues[id]; } diff --git a/Tests/EndToEndTests/UnitTests/ReaderTests/baseline.txt b/Tests/EndToEndTests/UnitTests/ReaderTests/baseline.txt index e0f20b4fa79d..99935c351e84 100644 --- a/Tests/EndToEndTests/UnitTests/ReaderTests/baseline.txt +++ b/Tests/EndToEndTests/UnitTests/ReaderTests/baseline.txt @@ -3,161 +3,161 @@ CPU info: Hardware threads: 1 Total Memory: 33476764 kB ------------------------------------------------------------------- -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKBinaryReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader WARNING: Exhausted all input expected for the current sequence while reading a floating point value at offset 43 in the input file (missing_trailing_newline.txt). attempt: Reached the maximum number of allowed errors while reading the input file (missing_trailing_newline.txt)., retrying 2-th time out of 5... WARNING: Exhausted all input expected for the current sequence while reading a floating point value at offset 43 in the input file (missing_trailing_newline.txt). @@ -167,22 +167,22 @@ attempt: Reached the maximum number of allowed errors while reading the input fi WARNING: Exhausted all input expected for the current sequence while reading a floating point value at offset 43 in the input file (missing_trailing_newline.txt). attempt: Reached the maximum number of allowed errors while reading the input file (missing_trailing_newline.txt)., retrying 5-th time out of 5... WARNING: Exhausted all input expected for the current sequence while reading a floating point value at offset 43 in the input file (missing_trailing_newline.txt). -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader WARNING: Exhausted all input expected for the current sequence while reading a floating point value at offset 43 in the input file (missing_trailing_newline.txt). WARNING: Exhausted all input expected for the current sequence while reading an input row at offset 43 in the input file (missing_trailing_newline.txt). Possibly, a trailing newline is missing. WARNING: Could not read a row (# 2) while loading sequence (id = 1) at offset 43 in the input file (missing_trailing_newline.txt). WARNING: Exhausted all input expected for the current sequence (id = 1) at offset 43 in the input file (missing_trailing_newline.txt), but only read 1 out of 2 expected rows. -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader WARNING: Empty input row at offset 19 in the input file (contains_blank_lines.txt). WARNING: Could not read a row (# 1) while loading sequence (id = 1) at offset 19 in the input file (contains_blank_lines.txt). attempt: Reached the maximum number of allowed errors while reading the input file (contains_blank_lines.txt)., retrying 2-th time out of 5... @@ -197,22 +197,22 @@ WARNING: Could not read a row (# 1) while loading sequence (id = 1) at offset 19 attempt: Reached the maximum number of allowed errors while reading the input file (contains_blank_lines.txt)., retrying 5-th time out of 5... WARNING: Empty input row at offset 19 in the input file (contains_blank_lines.txt). WARNING: Could not read a row (# 1) while loading sequence (id = 1) at offset 19 in the input file (contains_blank_lines.txt). -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader WARNING: Empty input row at offset 19 in the input file (contains_blank_lines.txt). WARNING: Could not read a row (# 2) while loading sequence (id = 1) at offset 19 in the input file (contains_blank_lines.txt). WARNING: Exhausted all input expected for the current sequence (id = 1) at offset 19 in the input file (contains_blank_lines.txt), but only read 1 out of 2 expected rows. WARNING: Maximum per-input number of samples for sequence (id = 1) at offset 19 in the input file (contains_blank_lines.txt) is less than expected (1 vs. 2). -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader WARNING: Input row at offset 27 in the input file (duplicate_inputs.txt) contains more samples than expected (3 vs. 2). WARNING: Input ('features2') contains more samples than expected (2 vs. 1) for sequence (id = 1) at offset 27 in the input file (duplicate_inputs.txt). attempt: Reached the maximum number of allowed errors while reading the input file (duplicate_inputs.txt)., retrying 2-th time out of 5... @@ -227,61 +227,61 @@ WARNING: Input ('features2') contains more samples than expected (2 vs. 1) for s attempt: Reached the maximum number of allowed errors while reading the input file (duplicate_inputs.txt)., retrying 5-th time out of 5... WARNING: Input row at offset 27 in the input file (duplicate_inputs.txt) contains more samples than expected (3 vs. 2). WARNING: Input ('features2') contains more samples than expected (2 vs. 1) for sequence (id = 1) at offset 27 in the input file (duplicate_inputs.txt). -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader WARNING: A dense sample at offset 6 in the input file (empty_samples.txt) has a sparse suffix (expected size = 10, actual size = 0). WARNING: A dense sample at offset 62 in the input file (empty_samples.txt) has a sparse suffix (expected size = 10, actual size = 5). WARNING: A dense sample at offset 478 in the input file (empty_samples.txt) has a sparse suffix (expected size = 10, actual size = 1). -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader/ +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data/CNTKTextFormatReader +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -294,17 +294,17 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 1, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 reading script file ./glob_0000.scp ... 948 entries total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries -biggrowablevectorarray: creating disk backup store at 'F:\cygwin64\tmp\CNT70F8.tmp' +biggrowablevectorarray: creating disk backup store at 'F:\cygwin64\tmp\CNT7688.tmp' minibatchframesourcemulti: reading 1 feature sets and 1 label sets.................................................................................................. minibatchframesourcemulti: read label set 0: 129 classes @@ -313,11 +313,11 @@ biggrowablevectorarray: disk backup store created, 252734 frames, 37748736 bytes minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses recoverblock: recovering feature block 0 [0..65535) minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 1, with 1 datapasses -biggrowablevectorarray: deleted disk backup store at 'F:\cygwin64\tmp\CNT70F8.tmp' -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +biggrowablevectorarray: deleted disk backup store at 'F:\cygwin64\tmp\CNT7688.tmp' +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -344,10 +344,10 @@ evaluate: reading 608 frames of An4/477/477/an257-mewl-b.mfc evaluate: reading 78 frames of An4/454/454/an70-meht-b.mfc evaluate: reading 228 frames of An4/254/254/cen6-ftmj-b.mfc evaluate: reading 218 frames of An4/946/946/cen6-mwhw-b.mfc -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -360,10 +360,10 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 886), data subset 0 of 1, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -376,17 +376,17 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 1, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -399,10 +399,10 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 886), data subset 0 of 1, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -436,10 +436,10 @@ getbatch: getting randomized frames [250..500] (250 frames out of 250 requested) minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 1, with 1 datapasses getbatch: getting randomized frames [500..750] (250 frames out of 250 requested) in sweep 0; chunks [0..0] -> chunk window [0..3) getbatch: getting randomized frames [750..1000] (250 frames out of 250 requested) in sweep 0; chunks [0..0] -> chunk window [0..3) -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -452,17 +452,17 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 1, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 reading script file ./glob_0000.scp ... 948 entries total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries -biggrowablevectorarray: creating disk backup store at 'F:\cygwin64\tmp\CNTEC33.tmp' +biggrowablevectorarray: creating disk backup store at 'F:\cygwin64\tmp\CNTEEF5.tmp' minibatchframesourcemulti: reading 1 feature sets and 1 label sets.................................................................................................. minibatchframesourcemulti: read label set 0: 129 classes @@ -471,11 +471,11 @@ biggrowablevectorarray: disk backup store created, 252734 frames, 37748736 bytes minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses recoverblock: recovering feature block 0 [0..65535) minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 1, with 1 datapasses -biggrowablevectorarray: deleted disk backup store at 'F:\cygwin64\tmp\CNTEC33.tmp' -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +biggrowablevectorarray: deleted disk backup store at 'F:\cygwin64\tmp\CNTEEF5.tmp' +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -502,10 +502,10 @@ evaluate: reading 608 frames of An4/477/477/an257-mewl-b.mfc evaluate: reading 78 frames of An4/454/454/an70-meht-b.mfc evaluate: reading 228 frames of An4/254/254/cen6-ftmj-b.mfc evaluate: reading 218 frames of An4/946/946/cen6-mwhw-b.mfc -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -518,10 +518,10 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 886), data subset 0 of 1, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -534,10 +534,10 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 1, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -549,10 +549,10 @@ label set 0: 129 classes minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: 316.0 utterances, 84244.7 frames minibatchiterator: epoch 0: frames [0..2000] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -586,10 +586,10 @@ getbatch: getting randomized frames [250..500] (250 frames out of 250 requested) minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 1, with 1 datapasses getbatch: getting randomized frames [500..750] (250 frames out of 250 requested) in sweep 0; chunks [0..0] -> chunk window [0..3) getbatch: getting randomized frames [750..1000] (250 frames out of 250 requested) in sweep 0; chunks [0..0] -> chunk window [0..3) -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -602,10 +602,10 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 0 of 2, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 0 of 2, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -618,10 +618,10 @@ minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: minibatchiterator: epoch 0: frames [0..500] (first utterance at frame 0), data subset 1 of 2, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms minibatchiterator: epoch 1: frames [500..1000] (first utterance at frame 500), data subset 1 of 2, with 1 datapasses -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -762,10 +762,10 @@ getbatch: getting utterances 32..32 (268 subset of 268 frames out of 250 request getbatch: getting utterances 33..33 (348 subset of 348 frames out of 250 requested) in sweep 0 getbatch: getting utterances 34..34 (258 subset of 258 frames out of 250 requested) in sweep 0 getbatch: getting utterances 35..35 (598 subset of 598 frames out of 220 requested) in sweep 0 -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -775,10 +775,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -788,10 +788,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -801,10 +801,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -814,10 +814,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -827,10 +827,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -840,10 +840,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -853,10 +853,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -866,10 +866,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -879,10 +879,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -894,10 +894,10 @@ label set 0: 129 classes minibatchutterancesource: 948 utterances grouped into 3 chunks, av. chunk size: 316.0 utterances, 84244.7 frames minibatchiterator: epoch 0: frames [0..2000] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses requiredata: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -907,10 +907,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -951,10 +951,10 @@ HTKChunkDescription::ReleaseData: release physical chunk 1 (341 utterances, 9015 HTKChunkDescription::ReleaseData: release physical chunk 1 (341 utterances, 90158 frames, 11900856 bytes) HTKChunkDescription::ReleaseData: release physical chunk 0 (328 utterances, 90234 frames, 11910888 bytes) HTKChunkDescription::ReleaseData: release physical chunk 0 (328 utterances, 90234 frames, 11910888 bytes) -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -998,27 +998,27 @@ HTKChunkDescription::ReleaseData: release physical chunk 1 (341 utterances, 9015 HTKChunkDescription::ReleaseData: release physical chunk 1 (341 utterances, 90158 frames, 11900856 bytes) HTKChunkDescription::ReleaseData: release physical chunk 0 (328 utterances, 90234 frames, 11910888 bytes) HTKChunkDescription::ReleaseData: release physical chunk 0 (328 utterances, 90234 frames, 11910888 bytes) -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 Reading script file ./glob_0000.scp ... 948 entries HTKDataDeserializer::HTKDataDeserializer: selected 948 utterances grouped into 3 chunks, average chunk size: 316.0 utterances, 84244.7 frames (for I/O: 316.0 utterances, 84244.7 frames) HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensional 'USER' with frame shift 10.0 ms -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 -Reading script file NonExistingScpFile ...Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Reading script file NonExistingScpFile ...Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -1028,10 +1028,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/Speech/AN4Corpus/v0 Current working directory is now: d:/TestPreparation/Speech/AN4Corpus/v0 @@ -1041,119 +1041,125 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 33-dimensio total 132 state names in state list ./state.list htkmlfreader: reading MLF file ./glob_0000.mlf ... total 948 entries MLFDataDeserializer::MLFDataDeserializer: 948 utterances with 252734 frames in 129 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data Sequence missing.jpg is not found in container images/simple.zip. -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/Data -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests/. -Current working directory is now: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/Data +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests/. +Current working directory is now: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/iVector Current working directory is now: d:/TestPreparation/iVector @@ -1171,10 +1177,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 40-dimensio total 9000 state names in state list ./labels.statelist htkmlfreader: reading MLF file ./labels.smlf ... total 4974 entries MLFDataDeserializer::MLFDataDeserializer: 4974 utterances with 1671772 frames in 9000 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/iVector Current working directory is now: d:/TestPreparation/iVector @@ -1199,10 +1205,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 100-dimensi total 9000 state names in state list ./labels.statelist htkmlfreader: reading MLF file ./labels.smlf ... total 4974 entries MLFDataDeserializer::MLFDataDeserializer: 4974 utterances with 1671772 frames in 9000 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/iVector Current working directory is now: d:/TestPreparation/iVector @@ -1227,10 +1233,10 @@ HTKDataDeserializer::HTKDataDeserializer: determined feature kind as 100-dimensi total 9000 state names in state list ./labels.statelist htkmlfreader: reading MLF file ./labels.smlf ... total 4974 entries MLFDataDeserializer::MLFDataDeserializer: 4974 utterances with 1671772 frames in 9000 classes -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Current working directory: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests -Executable path: C:/repo/cntk_github3/CNTK/x64/release -Test path: C:/repo/cntk_github3/CNTK/Tests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Current working directory: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Executable path: C:/repo/cntk_github6/CNTK/x64/release +Test path: C:/repo/cntk_github6/CNTK/Tests/UnitTests/ReaderTests Retrieving environment variable: CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY Set current path to: d:\TestPreparation/iVector Current working directory is now: d:/TestPreparation/iVector @@ -1241,7 +1247,7 @@ htkmlfreader: reading MLF file ./labels.smlf ... total 4974 entries ....................................................................................................feature set 0: 1669414 frames in 4963 out of 4963 utterances label set 0: 9000 classes minibatchutterancesource: 4963 utterances grouped into 19 chunks, av. chunk size: 261.2 utterances, 87863.9 frames -....................................................................................................Running 113 test cases... +....................................................................................................Running 114 test cases... feature set 1: 1669414 frames in 4963 out of 4963 utterances minibatchutterancesource: 4963 utterances grouped into 19 chunks, av. chunk size: 261.2 utterances, 87863.9 frames minibatchiterator: epoch 0: frames [0..400] (first utterance at frame 0), data subset 0 of 1, with 1 datapasses @@ -1257,15 +1263,15 @@ total 9000 state names in state list ./labels.statelist htkmlfreader: reading MLF file ./labels.smlf ... total 4974 entries MLFDataDeserializer::MLFDataDeserializer: 4974 utterances with 1671772 frames in 9000 classes Legacy configuration is used for truncated BPTT mode, please adapt the config to explicitly specify truncationLength. -Set current path to: C:/repo/cntk_github3/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests +Set current path to: C:/repo/cntk_github6/CNTK/Tests/EndToEndTests/UnitTests/ReaderTests Test module "ReaderTests" has passed with: - 113 test cases out of 113 passed - 16973519 assertions out of 16973519 passed + 114 test cases out of 114 passed + 16973520 assertions out of 16973520 passed Test suite "ReaderTestSuite" has passed with: - 91 test cases out of 91 passed - 16971334 assertions out of 16971334 passed + 92 test cases out of 92 passed + 16971335 assertions out of 16971335 passed Test case "ReaderTestSuite/CNTKBinaryReader_sparse_seq" has passed with: 1 assertion out of 1 passed @@ -1495,6 +1501,9 @@ Test module "ReaderTests" has passed with: Test case "ReaderTestSuite/ImageSimpleCompositeAndBase64" has passed with: 2 assertions out of 2 passed + Test case "ReaderTestSuite/InvalidImageSimpleCompositeAndBase64" has passed with: + 1 assertion out of 1 passed + Test case "ReaderTestSuite/ImageAndImageReaderSimple" has passed with: 1 assertion out of 1 passed diff --git a/Tests/UnitTests/ReaderTests/Data/InvalidBase64ImageReaderSimple_map.txt b/Tests/UnitTests/ReaderTests/Data/InvalidBase64ImageReaderSimple_map.txt new file mode 100644 index 000000000000..30c57ce49f42 --- /dev/null +++ b/Tests/UnitTests/ReaderTests/Data/InvalidBase64ImageReaderSimple_map.txt @@ -0,0 +1 @@ +0 /9k/4AAQSkZJRgABAQEAYABgAAD/4QAiRXhpZgAATU0AKgAAAAgAAQESAAMAAAABAAEAAAAAAAD/2wBDAAIBAQIBAQICAgICAgICAwUDAwMDAwYEBAMFBwYHBwcGBwcICQsJCAgKCAcHCg0KCgsMDAwMBwkODw0MDgsMDAz/2wBDAQICAgMDAwYDAwYMCAcIDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAwMDAz/wAARCAAIAAQDASIAAhEBAxEB/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQAAAF9AQIDAAQRBRIhMUEGE1FhByJxFDKBkaEII0KxwRVS0fAkM2JyggkKFhcYGRolJicoKSo0NTY3ODk6Q0RFRkdISUpTVFVWV1hZWmNkZWZnaGlqc3R1dnd4eXqDhIWGh4iJipKTlJWWl5iZmqKjpKWmp6ipqrKztLW2t7i5usLDxMXGx8jJytLT1NXW19jZ2uHi4+Tl5ufo6erx8vP09fb3+Pn6/8QAHwEAAwEBAQEBAQEBAQAAAAAAAAECAwQFBgcICQoL/8QAtREAAgECBAQDBAcFBAQAAQJ3AAECAxEEBSExBhJBUQdhcRMiMoEIFEKRobHBCSMzUvAVYnLRChYkNOEl8RcYGRomJygpKjU2Nzg5OkNERUZHSElKU1RVVldYWVpjZGVmZ2hpanN0dXZ3eHl6goOEhYaHiImKkpOUlZaXmJmaoqOkpaanqKmqsrO0tba3uLm6wsPExcbHyMnK0tPU1dbX2Nna4uPk5ebn6Onq8vP09fb3+Pn6/9oADAMBAAIRAxEAPwD+f+iiigD/2Q== diff --git a/Tests/UnitTests/ReaderTests/ImageReaderTests.cpp b/Tests/UnitTests/ReaderTests/ImageReaderTests.cpp index a583ed62f4c5..92b51d0882f7 100644 --- a/Tests/UnitTests/ReaderTests/ImageReaderTests.cpp +++ b/Tests/UnitTests/ReaderTests/ImageReaderTests.cpp @@ -82,7 +82,38 @@ BOOST_AUTO_TEST_CASE(ImageSimpleCompositeAndBase64) test( { L"MapFile=\"$RootDir$/Base64ImageReaderSimple_map.txt\"", - L"DeserializerType=\"Base64ImageDeserializer\"]]" + L"DeserializerType=\"Base64ImageDeserializer\"" + }); +}; + +BOOST_AUTO_TEST_CASE(InvalidImageSimpleCompositeAndBase64) +{ + auto test = [this](std::vector additionalParameters) + { + HelperRunReaderTest( + testDataPath() + "/Config/ImageReaderSimple_Config.cntk", + testDataPath() + "/Control/InvalidImageSimpleCompositeAndBase64_Control.txt", + testDataPath() + "/Control/InvalidImageSimpleCompositeAndBase64_Output.txt", + "Composite_Test", + "reader", + 4, + 4, + 1, + 1, + 1, + 0, + 1, + false, + false, + true, + additionalParameters); + }; + + test( + { + L"MapFile=\"$RootDir$/InvalidBase64ImageReaderSimple_map.txt\"", + L"DeserializerType=\"Base64ImageDeserializer\"" + L"maxErrors=4" }); }; From 99b4a4ef9f304397e9b5d3a3d5d77ee7673f677e Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 5 Jan 2017 15:51:47 +0100 Subject: [PATCH 019/120] This commit intentionally left empty. From 938df8c79bd64ece3351a94fcb0505b73f913a27 Mon Sep 17 00:00:00 2001 From: Chris Basoglu Date: Thu, 5 Jan 2017 09:55:14 -0800 Subject: [PATCH 020/120] Fix some documentation --- bindings/python/cntk/utils/__init__.py | 16 +++++++++------- bindings/python/cntk/utils/progress_print.py | 19 +++++++++++++------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/bindings/python/cntk/utils/__init__.py b/bindings/python/cntk/utils/__init__.py index 9162aff00af6..9202293bea09 100644 --- a/bindings/python/cntk/utils/__init__.py +++ b/bindings/python/cntk/utils/__init__.py @@ -474,10 +474,11 @@ class Value(cntk_py.Value): value (None or value that can be cast to NumPy array): the value to be converted dtype: data type (np.float32 or np.float64) - batch: batch input for `var`. It can be - * a pure Python structure (list of lists, ...), - * a list of NumPy arrays or SciPy sparse CSR matrices - * a :class:`Value` object (e.g. returned by :func:`one_hot`) + batch: batch input for `var`. + It can be: + * a pure Python structure (list of lists, ...), + * a list of NumPy arrays or SciPy sparse CSR matrices + * a :class:`Value` object (e.g. returned by :func:`one_hot`) seq_starts (list of `bool`s or None): if None, every sequence is treated as a new sequence. Otherwise, it is interpreted as a list of Booleans that tell whether a sequence is a new sequence (`True`) or a @@ -514,9 +515,10 @@ def create(var, batch, seq_starts=None, device=None, read_only=False): Args: var (:class:`~cntk.ops.variables.Variable`): input variable into which ``batch`` is passed - batch: batch input. It can be - * a single NumPy array denoting the full minibatch - * a list of NumPy arrays or SciPy sparse CSR matrices + batch: batch input. + It can be: + * a single NumPy array denoting the full minibatch + * a list of NumPy arrays or SciPy sparse CSR matrices seq_starts (list of `bool`s or None): if None, every sequence is treated as a new sequence. Otherwise, it is interpreted as a list of Booleans that tell whether a sequence is a new sequence (`True`) or a diff --git a/bindings/python/cntk/utils/progress_print.py b/bindings/python/cntk/utils/progress_print.py index 01e1a45f5977..91fa4ceb6993 100644 --- a/bindings/python/cntk/utils/progress_print.py +++ b/bindings/python/cntk/utils/progress_print.py @@ -18,12 +18,19 @@ class ProgressPrinter(object): ''' def __init__(self, freq=None, first=0, tag='', log_to_file=None, distributed_learner=None, gen_heartbeat=False, num_epochs=300): ''' - Constructor. The optional ``freq`` parameter determines how often - printing will occur. The value of 0 means an geometric - schedule (1,2,4,...). A value > 0 means a arithmetic schedule - (freq, 2*freq, 3*freq,...), and a value of None means no per-minibatch log. - set log_to_file if you want the output to go file instead of stdout. - set distributed_learner to your learner if you are using distibuted parallelism -- each rank's log will go to seperate file. + Constructor. + + Args: + freq (int or None, default None): determines how often + printing will occur. The value of 0 means an geometric + schedule (1,2,4,...). A value > 0 means a arithmetic schedule + (a log print for minibatch number: ``freq``, a log print for minibatch number: 2*``freq``, a log print for minibatch number: 3*``freq``,...), and a value of None means no per-minibatch log. + first (int, default 0): Only start logging after the minibatch number is greater or equal to ``first``. + tag (string, default EmptyString): prepend minibatch log lines with your own string + log_to_file (string or None, default None): if None, output log data to stdout. If a string is passed, the string is path to a file for log data. + distributed_learner (:class:`~cntk.distributed.DistributedLearner` or None, default None): Your learner if you are using distributed parallelism -- each rank's log will go to seperate file. + gen_heartbeat (bool, default False): If True output a progress message every 10 seconds or so to stdout. + num_epochs (int, default 300): The total number of epochs to be trained. Used for some metadata. This parameter is optional. ''' from sys import maxsize if freq is None: From 075db890ada00a00c6162523ac24d1c2ddee6b1c Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Wed, 4 Jan 2017 23:51:05 -0800 Subject: [PATCH 021/120] CNTK v2 library: Refactor Variable::VariableFields out of CNTKLibrary.h public header --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 108 +++--------------- .../API/CNTKLibraryInternals.h | 3 + .../CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj | 1 + .../CNTKv2LibraryDll.vcxproj.filters | 1 + Source/CNTKv2LibraryDll/PrimitiveFunction.h | 1 + Source/CNTKv2LibraryDll/Variable.cpp | 73 +++++++++++- Source/CNTKv2LibraryDll/Variable.h | 77 +++++++++++++ 7 files changed, 168 insertions(+), 96 deletions(-) create mode 100644 Source/CNTKv2LibraryDll/Variable.h diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 947c9eafc7c2..14c8a40c0e50 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -1693,22 +1693,22 @@ namespace CNTK /// /// Returns the shape of 'this' variable /// - const NDShape& Shape() const { return m_dataFields->m_shape; } + CNTK_API const NDShape& Shape() const; /// /// Returns the dynamic axes of 'this' variable /// - const std::vector& DynamicAxes() const { return m_dataFields->m_dynamicAxes; } + CNTK_API const std::vector& DynamicAxes() const; /// /// Returns the VariableKind of 'this' variable /// - VariableKind Kind() const { return m_dataFields->m_varKind; } + CNTK_API VariableKind Kind() const; /// /// Returns a boolean value indicating if 'this' variable denotes sparse data /// - bool IsSparse() const { return m_dataFields->m_isSparse; } + CNTK_API bool IsSparse() const; /// /// Returns a boolean value indicating if 'this' variable is an Input @@ -1738,12 +1738,12 @@ namespace CNTK /// /// Returns the name of 'this' variable /// - const std::wstring& Name() const { return m_dataFields->m_name; } + CNTK_API const std::wstring& Name() const; /// /// Returns the internally generated unique name of the variable /// - const std::wstring& Uid() const { return m_dataFields->m_uid; } + CNTK_API const std::wstring& Uid() const; /// /// Returns the Function object which 'this' variable is an output of. @@ -1754,12 +1754,12 @@ namespace CNTK /// /// Returns the DataType of the data that 'this' Variable symbolically represents /// - DataType GetDataType() const { return m_dataFields->m_dataType; } + CNTK_API DataType GetDataType() const; /// /// Returns a boolean value indicating if gradient computation is enabled for this variable. /// - bool NeedsGradient() const { return m_dataFields->m_needsGradient; } + CNTK_API bool NeedsGradient() const; protected: #ifdef SWIG @@ -1787,13 +1787,7 @@ namespace CNTK CNTK_API Variable(const NDShape& shape, VariableKind varType, ::CNTK::DataType dataType, const NDArrayViewPtr& value, bool needsGradient, const std::vector& dynamicAxes, bool isSparse, const std::wstring& name, const std::wstring& uid); private: - Variable Clone() const - { - Variable clonedVariable; - clonedVariable.m_dataFields = m_dataFields->Clone(); - - return clonedVariable; - } + CNTK_API Variable Clone() const; CNTK_API virtual Dictionary Serialize() const override; @@ -1807,71 +1801,6 @@ namespace CNTK void SetOwner(Function* ownerFunction); private: - struct VariableFields final : public std::enable_shared_from_this - { - friend class CompositeFunction; - - NDShape m_shape; - VariableKind m_varKind; - ::CNTK::DataType m_dataType; - Function* m_ownerFunction; // Variable does not keep the Function alive - std::unique_ptr m_initValueFlag; - NDArrayViewPtr m_value; - std::unique_ptr m_valueInitializer; - std::unique_ptr m_valueInitializationDevice; - bool m_needsGradient; - std::wstring m_name; - std::vector m_dynamicAxes; - bool m_isSparse; - std::wstring m_uid; - std::atomic m_valueTimeStamp; - - VariableFields(const NDShape& shape, VariableKind varType, ::CNTK::DataType type, Function* ownerFunction, const NDArrayViewPtr& value, bool needsGradient, const std::vector& dynamicAxes, bool isSparse, const std::wstring& name, const std::wstring& uid) - : m_shape(shape), m_varKind(varType), m_dataType(type), m_ownerFunction(ownerFunction), m_value(value), m_needsGradient(needsGradient), m_dynamicAxes(dynamicAxes), m_isSparse(isSparse), m_name(name), m_uid(uid), m_valueTimeStamp(0) - { - if (value && (type != value->GetDataType())) - InvalidArgument("The DataType of the Parameter/Constant Variable does not match the DataType of the associated Value"); - - // Validate that each of the dynamic axes are unique - std::unordered_set uniqueDynamicAxis; - for (auto& currentDynamicAxis : dynamicAxes) - { - auto retVal = uniqueDynamicAxis.insert(currentDynamicAxis); - if (!retVal.second) - InvalidArgument("Dynamic axis named %S is specified more than once for Variable object", currentDynamicAxis.Name().c_str()); - } - } - - std::shared_ptr Clone() const - { - if (m_ownerFunction != nullptr) - InvalidArgument("Output variables cannot be cloned"); - - auto clone = MakeSharedObject(m_shape, - m_varKind, - m_dataType, - m_ownerFunction, - (m_value) ? m_value->DeepClone() : nullptr, - m_needsGradient, - m_dynamicAxes, - m_isSparse, - m_name, - Internal::GenerateUid(m_varKind)); - - if (m_valueInitializer) - clone->SetValueInitialization(*m_valueInitializer, *m_valueInitializationDevice); - - return clone; - } - - CNTK_API void SetValueInitialization(const ParameterInitializer& initializationConfig, const DeviceDescriptor& device); - - private: - // Disallow copy and move construction and assignment - VariableFields(const VariableFields&) = delete; VariableFields& operator=(const VariableFields& other) = delete; VariableFields(VariableFields&&) = delete; VariableFields& operator=(VariableFields&&) = delete; - }; - typedef std::shared_ptr VariableFieldsPtr; - #ifdef SWIGCSHARP public: // Todo: a better way to get hash value? @@ -2061,11 +1990,7 @@ namespace CNTK /// /// Construct a constant of specified shape whose contents are initialized using the specified initializer /// - Parameter(const NDShape& shape, DataType dataType, const ParameterInitializer& initializer, const DeviceDescriptor& device = DeviceDescriptor::UseDefaultDevice(), const std::wstring& name = L"") - : Variable(shape, VariableKind::Parameter, dataType, nullptr, true, {}, name, Internal::GenerateUid(VariableKind::Parameter)) - { - m_dataFields->SetValueInitialization(initializer, device); - } + CNTK_API Parameter(const NDShape& shape, DataType dataType, const ParameterInitializer& initializer, const DeviceDescriptor& device = DeviceDescriptor::UseDefaultDevice(), const std::wstring& name = L""); /// /// DownCast a Variable to a Parameter. Only allowed if the VariableKind is Parameter and throws an exception otherwise. @@ -2095,12 +2020,9 @@ namespace CNTK RecordValueUpdate(); } - size_t CurrentValueTimeStamp() const { return m_dataFields->m_valueTimeStamp.load(); } + CNTK_API size_t CurrentValueTimeStamp() const; - void RecordValueUpdate() - { - m_dataFields->m_valueTimeStamp++; - } + CNTK_API void RecordValueUpdate(); private: explicit Parameter(const NDArrayViewPtr& value, const std::wstring& name, const std::wstring& uid) @@ -2192,11 +2114,7 @@ namespace CNTK /// /// Construct a constant of specified shape whose contents are initialized using the specified initializer /// - Constant(const NDShape& shape, DataType dataType, const ParameterInitializer& initializer, const DeviceDescriptor& device = DeviceDescriptor::UseDefaultDevice(), const std::wstring& name = L"") - : Variable(shape, VariableKind::Constant, dataType, nullptr, false, {}, name, Internal::GenerateUid(VariableKind::Parameter)) - { - m_dataFields->SetValueInitialization(initializer, device); - } + CNTK_API Constant(const NDShape& shape, DataType dataType, const ParameterInitializer& initializer, const DeviceDescriptor& device = DeviceDescriptor::UseDefaultDevice(), const std::wstring& name = L""); }; // Implementation note: The Variable type is a value type and not polymorphic in nature. diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h b/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h index d1b3643dc5ec..be8602023ad2 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h @@ -205,6 +205,9 @@ namespace CNTK class DistributedLearner; typedef std::shared_ptr DistributedLearnerPtr; + struct VariableFields; + typedef std::shared_ptr VariableFieldsPtr; + namespace Internal { CNTK_API FunctionPtr IsWithin(const Variable& operand, int offset, const std::wstring& name = L""); diff --git a/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj b/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj index 5999bfbfb7fe..ab1c3b5cd590 100644 --- a/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj +++ b/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj @@ -149,6 +149,7 @@ + diff --git a/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj.filters b/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj.filters index d2868514d1bb..a7ce7f1c5e46 100644 --- a/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj.filters +++ b/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj.filters @@ -50,6 +50,7 @@ + diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.h b/Source/CNTKv2LibraryDll/PrimitiveFunction.h index 25e7403e78a8..8b41593d0ea4 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.h +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.h @@ -11,6 +11,7 @@ #include "Utils.h" #include "ConvolveGeometry.h" #include "ConvolutionalNodes.h" +#include "Variable.h" namespace std { diff --git a/Source/CNTKv2LibraryDll/Variable.cpp b/Source/CNTKv2LibraryDll/Variable.cpp index f088a8b0ab45..7c594b2fe0fc 100644 --- a/Source/CNTKv2LibraryDll/Variable.cpp +++ b/Source/CNTKv2LibraryDll/Variable.cpp @@ -5,6 +5,7 @@ #include "stdafx.h" #include "CNTKLibrary.h" +#include "Variable.h" #include "CompositeFunction.h" #include "Serialization.h" #include "InputAndParamNodes.h" @@ -16,6 +17,54 @@ namespace CNTK { } + const NDShape& Variable::Shape() const + { + return m_dataFields->m_shape; + } + + const std::vector& Variable::DynamicAxes() const + { + return m_dataFields->m_dynamicAxes; + } + + VariableKind Variable::Kind() const + { + return m_dataFields->m_varKind; + } + + bool Variable::IsSparse() const + { + return m_dataFields->m_isSparse; + } + + const std::wstring& Variable::Name() const + { + return m_dataFields->m_name; + } + + const std::wstring& Variable::Uid() const + { + return m_dataFields->m_uid; + } + + DataType Variable::GetDataType() const + { + return m_dataFields->m_dataType; + } + + bool Variable::NeedsGradient() const + { + return m_dataFields->m_needsGradient; + } + + Variable Variable::Clone() const + { + Variable clonedVariable; + clonedVariable.m_dataFields = m_dataFields->Clone(); + + return clonedVariable; + } + FunctionPtr Variable::Owner() const { if (m_dataFields->m_ownerFunction != nullptr) @@ -122,7 +171,7 @@ namespace CNTK static const std::wstring KernelWidthAttributeName = L"kernelWidth"; static const std::wstring KernelHeightAttributeName = L"kernelHeight"; - void Variable::VariableFields::SetValueInitialization(const ParameterInitializer& initializationConfig, const DeviceDescriptor& device) + void VariableFields::SetValueInitialization(const ParameterInitializer& initializationConfig, const DeviceDescriptor& device) { if (m_value != nullptr) LogicError("Value initialization config cannot be set if a value already exists"); @@ -402,4 +451,26 @@ namespace CNTK return Variable(shape, kind, dataType, nullptr, needsGradient, dynamicAxis, isSparse, name, uid); } + + Parameter::Parameter(const NDShape& shape, DataType dataType, const ParameterInitializer& initializer, const DeviceDescriptor& device, const std::wstring& name) + : Variable(shape, VariableKind::Parameter, dataType, nullptr, true, {}, name, Internal::GenerateUid(VariableKind::Parameter)) + { + m_dataFields->SetValueInitialization(initializer, device); + } + + size_t Parameter::CurrentValueTimeStamp() const + { + return m_dataFields->m_valueTimeStamp.load(); + } + + void Parameter::RecordValueUpdate() + { + m_dataFields->m_valueTimeStamp++; + } + + Constant::Constant(const NDShape& shape, DataType dataType, const ParameterInitializer& initializer, const DeviceDescriptor& device, const std::wstring& name) + : Variable(shape, VariableKind::Constant, dataType, nullptr, false, {}, name, Internal::GenerateUid(VariableKind::Constant)) + { + m_dataFields->SetValueInitialization(initializer, device); + } } diff --git a/Source/CNTKv2LibraryDll/Variable.h b/Source/CNTKv2LibraryDll/Variable.h new file mode 100644 index 000000000000..271d3ca16e1f --- /dev/null +++ b/Source/CNTKv2LibraryDll/Variable.h @@ -0,0 +1,77 @@ +// +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +// + +#pragma once + +#include "stdafx.h" +#include "CNTKLibrary.h" +#include + +namespace CNTK +{ + struct VariableFields final : public std::enable_shared_from_this + { + friend class CompositeFunction; + + NDShape m_shape; + VariableKind m_varKind; + ::CNTK::DataType m_dataType; + Function* m_ownerFunction; // Variable does not keep the Function alive + std::unique_ptr m_initValueFlag; + NDArrayViewPtr m_value; + std::unique_ptr m_valueInitializer; + std::unique_ptr m_valueInitializationDevice; + bool m_needsGradient; + std::wstring m_name; + std::vector m_dynamicAxes; + bool m_isSparse; + std::wstring m_uid; + std::atomic m_valueTimeStamp; + + VariableFields(const NDShape& shape, VariableKind varType, ::CNTK::DataType type, Function* ownerFunction, const NDArrayViewPtr& value, bool needsGradient, const std::vector& dynamicAxes, bool isSparse, const std::wstring& name, const std::wstring& uid) + : m_shape(shape), m_varKind(varType), m_dataType(type), m_ownerFunction(ownerFunction), m_value(value), m_needsGradient(needsGradient), m_dynamicAxes(dynamicAxes), m_isSparse(isSparse), m_name(name), m_uid(uid), m_valueTimeStamp(0) + { + if (value && (type != value->GetDataType())) + InvalidArgument("The DataType of the Parameter/Constant Variable does not match the DataType of the associated Value"); + + // Validate that each of the dynamic axes are unique + std::unordered_set uniqueDynamicAxis; + for (auto& currentDynamicAxis : dynamicAxes) + { + auto retVal = uniqueDynamicAxis.insert(currentDynamicAxis); + if (!retVal.second) + InvalidArgument("Dynamic axis named %S is specified more than once for Variable object", currentDynamicAxis.Name().c_str()); + } + } + + std::shared_ptr Clone() const + { + if (m_ownerFunction != nullptr) + InvalidArgument("Output variables cannot be cloned"); + + auto clone = MakeSharedObject(m_shape, + m_varKind, + m_dataType, + m_ownerFunction, + (m_value) ? m_value->DeepClone() : nullptr, + m_needsGradient, + m_dynamicAxes, + m_isSparse, + m_name, + Internal::GenerateUid(m_varKind)); + + if (m_valueInitializer) + clone->SetValueInitialization(*m_valueInitializer, *m_valueInitializationDevice); + + return clone; + } + + CNTK_API void SetValueInitialization(const ParameterInitializer& initializationConfig, const DeviceDescriptor& device); + + private: + // Disallow copy and move construction and assignment + VariableFields(const VariableFields&) = delete; VariableFields& operator=(const VariableFields& other) = delete; VariableFields(VariableFields&&) = delete; VariableFields& operator=(VariableFields&&) = delete; + }; +} From fe24966f7e240fe8916e71b783712b7c44e02699 Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Thu, 5 Jan 2017 02:22:42 -0800 Subject: [PATCH 022/120] CNTK v2 library: Change BlockFunction implementation to store the mapping of block function arguments and outputs in the variables themselves instead of maps on teh side --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 9 +- Source/CNTKv2LibraryDll/BlockFunction.h | 95 +++++++++++++------ Source/CNTKv2LibraryDll/CompositeFunction.cpp | 18 ++-- Source/CNTKv2LibraryDll/Function.cpp | 4 +- Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 2 +- Source/CNTKv2LibraryDll/Variable.cpp | 15 ++- Source/CNTKv2LibraryDll/Variable.h | 3 + .../CNTK_202_Language_Understanding.ipynb | 23 ++--- 8 files changed, 109 insertions(+), 60 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 14c8a40c0e50..64c91d298b86 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -1787,6 +1787,8 @@ namespace CNTK CNTK_API Variable(const NDShape& shape, VariableKind varType, ::CNTK::DataType dataType, const NDArrayViewPtr& value, bool needsGradient, const std::vector& dynamicAxes, bool isSparse, const std::wstring& name, const std::wstring& uid); private: + CNTK_API const Variable& BlockFunctionVariableMapping() const; + CNTK_API Variable Clone() const; CNTK_API virtual Dictionary Serialize() const override; @@ -2346,7 +2348,10 @@ namespace CNTK /// to the Variables that they are bound to in the outer graph of Functions that this /// block Function is part of. /// - CNTK_API const std::vector>& BlockArgumentsMapping() const; + std::vector> BlockArgumentsMapping() const + { + return *BlockArgumentsMappingImpl().get(); + } /// /// Returns all Input variables of 'this' Function. @@ -2469,6 +2474,8 @@ namespace CNTK private: + CNTK_API std::shared_ptr>> BlockArgumentsMappingImpl() const; + template std::vector FilteredInputs(FilterFunction&& filterFunc) const { diff --git a/Source/CNTKv2LibraryDll/BlockFunction.h b/Source/CNTKv2LibraryDll/BlockFunction.h index ac65e2df9be8..1cabc503f5d9 100644 --- a/Source/CNTKv2LibraryDll/BlockFunction.h +++ b/Source/CNTKv2LibraryDll/BlockFunction.h @@ -16,7 +16,7 @@ namespace CNTK public: BlockFunction(FunctionPtr&& composite, const std::vector>& argumentsMap, const std::wstring& blockOpName, Dictionary&& attributes, const std::wstring& blockName = L"", const std::wstring& uid = GenerateUid(PrimitiveOpType::Block)) : PrimitiveFunction(DetermineInputs(composite, argumentsMap, blockName), DetermineOutputs(composite, blockName), std::move(attributes), blockName, uid), - m_composite(composite), m_blockOpName(blockOpName), m_compositeArgumentsMap(argumentsMap) + m_composite(composite), m_blockOpName(blockOpName) { auto updatedOutputs = GetOutputVariables(true); auto currentOutputs = Outputs(); @@ -30,27 +30,64 @@ namespace CNTK auto compositeOutputs = composite->Outputs(); for (size_t i = 0; i < currentOutputs.size(); ++i) - m_compositeOutputsMap.insert({ currentOutputs[i], compositeOutputs[i] }); + currentOutputs[i].m_dataFields->m_blockFunctionVariableMapping = compositeOutputs[i]; } - virtual const std::wstring& OpName() const override + virtual const std::wstring& OpName() const override { return m_blockOpName; } + + const FunctionPtr& Composite() const { return m_composite; } + + // Mapping from each argument of the composite underlying the block to the corresponding Variable it is mapped to + std::vector> CompositeArgumentsMap() const { - return m_blockOpName; + std::unordered_map argumentsMappingAsMap; + auto arguments = m_composite->Arguments(); + for (auto argument : arguments) + { + if (argument.BlockFunctionVariableMapping() == Variable()) + LogicError("BlockFunction (%S) with OpName (%S) does not have a mapping for argument (%S)", Name().c_str(), OpName().c_str(), argument.Name().c_str()); + + argumentsMappingAsMap[argument] = argument.BlockFunctionVariableMapping(); + } + + std::vector> argumentsMap; + auto blockInputs = Inputs(); + for (auto blockInput : blockInputs) + { + auto iter = std::find_if(argumentsMappingAsMap.begin(), argumentsMappingAsMap.end(), [&blockInput](const std::pair& entry) {return entry.second == blockInput; }); + if (iter != argumentsMappingAsMap.end()) + argumentsMap.push_back({iter->first, iter->second}); + } + + return argumentsMap; } - const FunctionPtr& Composite() const { return m_composite; } - const std::vector>& CompositeArgumentsMap() const { return m_compositeArgumentsMap; } - const std::unordered_map& CompositeOutputsMap() const { return m_compositeOutputsMap; } + // Mapping from each output of the block to the corresponding output of underlying composite + std::unordered_map CompositeOutputsMap() const + { + std::unordered_map outputsMap; + auto outputs = Outputs(); + for (auto output : outputs) + { + if (output.BlockFunctionVariableMapping() == Variable()) + LogicError("BlockFunction (%S) with OpName (%S) does not have a mapping for output (%S)", Name().c_str(), OpName().c_str(), output.Name().c_str()); + + outputsMap[output] = output.BlockFunctionVariableMapping(); + } + + return outputsMap; + } protected: virtual void OnPlaceholdersReplaced(const std::unordered_map& placeholderReplacements, std::unordered_set& replacedPlaceholders) override { // Substitute any placeholder replacements in the arguments map - for (auto& argMapping : m_compositeArgumentsMap) + auto arguments = m_composite->Arguments(); + for (auto argument : arguments) { - if (replacedPlaceholders.find(argMapping.second) != replacedPlaceholders.end()) - argMapping.second = placeholderReplacements.at(argMapping.second); + if (replacedPlaceholders.find(argument.BlockFunctionVariableMapping()) != replacedPlaceholders.end()) + argument.m_dataFields->m_blockFunctionVariableMapping = placeholderReplacements.at(argument.BlockFunctionVariableMapping()); } } @@ -98,7 +135,10 @@ namespace CNTK // We now append the mapped arguments of the composite to the block inputs in the order of the map // instead of the original order they appear in the composite itself for (auto argumentMapping : argumentsMap) + { + argumentMapping.first.m_dataFields->m_blockFunctionVariableMapping = argumentMapping.second; blockFunctionInputs.push_back(argumentMapping.second); + } return blockFunctionInputs; } @@ -107,23 +147,25 @@ namespace CNTK { // We determine the outputs by replacing the arguments of the composite with new placeholders with updated // shape etc. information matching the corresponding mapped input - std::vector> newArgumentsMap; + auto currentArguments = m_composite->Arguments(); std::unordered_map replacementMap; - for (auto argMapping : m_compositeArgumentsMap) + for (auto currentArgument : currentArguments) { - auto newArgument = PlaceholderVariable(argMapping.second.Shape(), argMapping.second.GetDataType(), argMapping.second.Name(), argMapping.second.DynamicAxes()); - newArgumentsMap.push_back({ newArgument, argMapping.second }); - replacementMap.insert({ argMapping.first, newArgument }); + auto currentArgumentMapping = currentArgument.BlockFunctionVariableMapping(); + auto newArgument = PlaceholderVariable(currentArgumentMapping.Shape(), currentArgumentMapping.GetDataType(), currentArgumentMapping.Name(), currentArgumentMapping.DynamicAxes()); + newArgument.m_dataFields->m_blockFunctionVariableMapping = currentArgumentMapping; + + replacementMap.insert({ currentArgument, newArgument }); } m_composite->ReplacePlaceholders(replacementMap); - m_compositeArgumentsMap = std::move(newArgumentsMap); // Substitute any placeholder replacements in the outputs map - for (auto& outputMapping : m_compositeOutputsMap) + auto outputs = Outputs(); + for (auto output : outputs) { - if (replacementMap.find(outputMapping.second) != replacementMap.end()) - outputMapping.second = replacementMap.at(outputMapping.second); + if (replacementMap.find(output.BlockFunctionVariableMapping()) != replacementMap.end()) + output.m_dataFields->m_blockFunctionVariableMapping = replacementMap.at(output.BlockFunctionVariableMapping()); } std::vector blockFunctionOutputs; @@ -139,7 +181,12 @@ namespace CNTK std::vector blockFunctionOutputs; auto compositeOutputs = composite->Outputs(); for (auto compositeOutput : compositeOutputs) - blockFunctionOutputs.push_back(OutputVariable(compositeOutput.Shape(), compositeOutput.GetDataType(), compositeOutput.DynamicAxes(), blockName)); + { + auto output = OutputVariable(compositeOutput.Shape(), compositeOutput.GetDataType(), compositeOutput.DynamicAxes(), blockName); + output.m_dataFields->m_blockFunctionVariableMapping = compositeOutput; + + blockFunctionOutputs.push_back(output); + } return blockFunctionOutputs; } @@ -148,14 +195,6 @@ namespace CNTK FunctionPtr m_composite; std::wstring m_blockOpName; - // Mapping from each argument of the composite underlying the block - // to the corresponding Variable it is mapped to - std::vector> m_compositeArgumentsMap; - - // Mapping from each output of the block to the corresponding - // output of underlying composite - std::unordered_map m_compositeOutputsMap; - // Increasing s_serializationVersion every time we add more ops allows us to print // a more meaningful message when trying to load a new model with a stale binary. static const size_t s_serializationVersion = 1; diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.cpp b/Source/CNTKv2LibraryDll/CompositeFunction.cpp index d1713ce2077f..a97db537f569 100644 --- a/Source/CNTKv2LibraryDll/CompositeFunction.cpp +++ b/Source/CNTKv2LibraryDll/CompositeFunction.cpp @@ -833,12 +833,11 @@ namespace CNTK // For block function, map each argument placeholder of the underlying composite to // the computation node corresponding to the block input that the argument placeholder // of the composite is mapped to. - auto& compositeArgumentsMap = blockFunction->CompositeArgumentsMap(); - for (auto& compositeArgumentMapping : compositeArgumentsMap) - variableToNodeMap[compositeArgumentMapping.first] = variableToNodeMap.at(compositeArgumentMapping.second); + auto compositeArguments = blockFunction->Composite()->Arguments(); + for (auto compositeArgument : compositeArguments) + variableToNodeMap[compositeArgument] = variableToNodeMap.at(compositeArgument.BlockFunctionVariableMapping()); - auto& compositeOutputsMap = blockFunction->CompositeOutputsMap(); - return GetNode(compositeOutputsMap.at(variable), network, builder, variableToNodeMap, isVariableRootMap); + return GetNode(variable.BlockFunctionVariableMapping(), network, builder, variableToNodeMap, isVariableRootMap); } else computationNodePtr = CreateComputationNode(variable, function, inputNodes, network, variableToNodeMap); @@ -892,11 +891,12 @@ namespace CNTK // since for recurrent inputs, the mappings are not fully established the first time std::function PatchBlockArgumentsMapping; PatchBlockArgumentsMapping = [this, &PatchBlockArgumentsMapping](const FunctionPtr& function) { - if (function->IsBlock()) + BlockFunction* blockFunction = dynamic_cast(function.get()); + if (blockFunction) { - auto& compositeArgumentsMap = function->BlockArgumentsMapping(); - for (auto& compositeArgumentMapping : compositeArgumentsMap) - m_variableToNodeMap[compositeArgumentMapping.first] = m_variableToNodeMap.at(compositeArgumentMapping.second); + auto compositeArguments = blockFunction->Composite()->Arguments(); + for (auto compositeArgument : compositeArguments) + m_variableToNodeMap[compositeArgument] = m_variableToNodeMap.at(compositeArgument.BlockFunctionVariableMapping()); PreorderTraverseFunctions(function->BlockComposite()->RootFunction(), PatchBlockArgumentsMapping); } diff --git a/Source/CNTKv2LibraryDll/Function.cpp b/Source/CNTKv2LibraryDll/Function.cpp index c0c34cbf7e45..072abcf15b35 100644 --- a/Source/CNTKv2LibraryDll/Function.cpp +++ b/Source/CNTKv2LibraryDll/Function.cpp @@ -83,13 +83,13 @@ namespace CNTK return blockFunction->Composite(); } - const std::vector>& Function::BlockArgumentsMapping() const + std::shared_ptr>> Function::BlockArgumentsMappingImpl() const { if (!IsBlock()) InvalidArgument("Function::BlockArgumentsMapping() cannot be called for a Function which is not a block"); auto blockFunction = dynamic_cast(this); - return blockFunction->CompositeArgumentsMap(); + return std::shared_ptr>>(new std::vector>(std::move(blockFunction->CompositeArgumentsMap())), [](std::vector>* ptr) { delete ptr; }); } /*static*/ void Function::ReplacePlaceholderInPlace(Variable& var, diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index aa52af753910..6f06c405fd10 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -635,7 +635,7 @@ namespace CNTK dict[blockFunctionCompositeKey] = blockCompositeFunc->SerializeBlockComposite(); dict[blockFunctionOpNameKey] = OpName(); - auto& blockArgumentsMap = BlockArgumentsMapping(); + const auto& blockArgumentsMap = BlockArgumentsMapping(); std::vector serializedArgumentsMapKeys; std::vector serializedArgumentsMapValues; for (auto argumentMapping : blockArgumentsMap) diff --git a/Source/CNTKv2LibraryDll/Variable.cpp b/Source/CNTKv2LibraryDll/Variable.cpp index 7c594b2fe0fc..ffd75bb39863 100644 --- a/Source/CNTKv2LibraryDll/Variable.cpp +++ b/Source/CNTKv2LibraryDll/Variable.cpp @@ -65,6 +65,11 @@ namespace CNTK return clonedVariable; } + const Variable& Variable::BlockFunctionVariableMapping() const + { + return m_dataFields->m_blockFunctionVariableMapping; + } + FunctionPtr Variable::Owner() const { if (m_dataFields->m_ownerFunction != nullptr) @@ -366,13 +371,12 @@ namespace CNTK dict[uidKey] = Uid(); dict[kindKey] = static_cast(Kind()); dict[dataTypeKey] = static_cast(GetDataType()); - const auto& dynamicAxis = DynamicAxes(); + const auto& dynamicAxes = DynamicAxes(); vector dictionaryValueVector; - dictionaryValueVector.reserve(dynamicAxis.size()); - for (const auto& axis : dynamicAxis) - { + dictionaryValueVector.reserve(dynamicAxes.size()); + for (const auto& axis : dynamicAxes) dictionaryValueVector.push_back(axis); - } + dict[dynamicAxisKey] = dictionaryValueVector; dict[isSparseKey] = IsSparse(); dict[nameKey] = Name(); @@ -389,6 +393,7 @@ namespace CNTK // TODO: add a dictionary value constructor with an rvalue parameter. dict[valueKey] = DictionaryValue(*value); } + return dict; } diff --git a/Source/CNTKv2LibraryDll/Variable.h b/Source/CNTKv2LibraryDll/Variable.h index 271d3ca16e1f..ebfcabc9b084 100644 --- a/Source/CNTKv2LibraryDll/Variable.h +++ b/Source/CNTKv2LibraryDll/Variable.h @@ -29,6 +29,7 @@ namespace CNTK bool m_isSparse; std::wstring m_uid; std::atomic m_valueTimeStamp; + Variable m_blockFunctionVariableMapping; VariableFields(const NDShape& shape, VariableKind varType, ::CNTK::DataType type, Function* ownerFunction, const NDArrayViewPtr& value, bool needsGradient, const std::vector& dynamicAxes, bool isSparse, const std::wstring& name, const std::wstring& uid) : m_shape(shape), m_varKind(varType), m_dataType(type), m_ownerFunction(ownerFunction), m_value(value), m_needsGradient(needsGradient), m_dynamicAxes(dynamicAxes), m_isSparse(isSparse), m_name(name), m_uid(uid), m_valueTimeStamp(0) @@ -51,6 +52,8 @@ namespace CNTK if (m_ownerFunction != nullptr) InvalidArgument("Output variables cannot be cloned"); + // Note: We do not clone m_blockFunctionVariableMapping + auto clone = MakeSharedObject(m_shape, m_varKind, m_dataType, diff --git a/Tutorials/CNTK_202_Language_Understanding.ipynb b/Tutorials/CNTK_202_Language_Understanding.ipynb index 91b867d3cddb..cc59c27ee876 100644 --- a/Tutorials/CNTK_202_Language_Understanding.ipynb +++ b/Tutorials/CNTK_202_Language_Understanding.ipynb @@ -211,8 +211,6 @@ "# model dimensions\n", "input_dim = vocab_size\n", "label_dim = num_labels\n", - "features_var = Input(vocab_size, name='features')\n", - "labels_var = Input(num_labels, name='labels')\n", "emb_dim = 150\n", "hidden_dim = 300\n", "\n", @@ -364,10 +362,8 @@ " # here (query, slot_labels) -> (ce, errs)\n", " criterion = create_criterion_function(model)\n", "\n", - " labels_placeholder = next(placeholder for placeholder in criterion.placeholders if placeholder.name == 'labels')\n", - " features_placeholder = next(placeholder for placeholder in criterion.placeholders if placeholder.name != 'labels')\n", - " criterion.replace_placeholders({features_placeholder: features_var,\n", - " labels_placeholder: labels_var})\n", + " criterion.replace_placeholders({criterion.placeholders[0]: Input(vocab_size),\n", + " criterion.placeholders[1]: Input(num_labels)})\n", "\n", " # training config\n", " epoch_size = 18000 # 18000 samples is half the dataset size \n", @@ -405,11 +401,11 @@ " epoch_end = (epoch+1) * epoch_size\n", " while t < epoch_end: # loop over minibatches on the epoch\n", " data = reader.next_minibatch(minibatch_size, input_map={ # fetch minibatch\n", - " features_var: reader.streams.query,\n", - " labels_var: reader.streams.slot_labels\n", + " criterion.arguments[0]: reader.streams.query,\n", + " criterion.arguments[1]: reader.streams.slot_labels\n", " })\n", " trainer.train_minibatch(data) # update model with it\n", - " t += data[labels_var].num_samples # samples so far\n", + " t += data[criterion.arguments[1]].num_samples # samples so far\n", " progress_printer.update_with_trainer(trainer, with_metric=True) # log progress\n", " loss, metric, actual_samples = progress_printer.epoch_summary(with_metric=True)\n", "\n", @@ -488,8 +484,7 @@ "source": [ "def evaluate(reader, model):\n", " criterion = create_criterion_function(model)\n", - " labels_placeholder = next(placeholder for placeholder in criterion.placeholders if placeholder.name == 'labels')\n", - " criterion.replace_placeholders({labels_placeholder: labels_var})\n", + " criterion.replace_placeholders({criterion.placeholders[0]: Input(num_labels)})\n", "\n", " # process minibatches and perform evaluation\n", " lr_schedule = learning_rate_schedule(1, UnitType.minibatch)\n", @@ -502,13 +497,13 @@ " while True:\n", " minibatch_size = 1000\n", " data = reader.next_minibatch(minibatch_size, input_map={ # fetch minibatch\n", - " features_var: reader.streams.query,\n", - " labels_var: reader.streams.slot_labels\n", + " criterion.arguments[0]: reader.streams.query,\n", + " criterion.arguments[1]: reader.streams.slot_labels\n", " })\n", " if not data: # until we hit the end\n", " break\n", " metric = evaluator.test_minibatch(data)\n", - " progress_printer.update(0, data[labels_var].num_samples, metric) # log progress\n", + " progress_printer.update(0, data[criterion.arguments[1]].num_samples, metric) # log progress\n", " loss, metric, actual_samples = progress_printer.epoch_summary(with_metric=True)\n", "\n", " return loss, metric" From 4f6bff84c9b309c53e06ab6ba93aed3a8d030534 Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Thu, 5 Jan 2017 03:18:55 -0800 Subject: [PATCH 023/120] CNTK v2 library: Fix a bug in where node dynamic axis determination where multiple where applications on same condition were yielding different dynamic axes --- Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 12 ++++++++- .../python/cntk/ops/tests/reshaping_test.py | 27 +++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index 6f06c405fd10..88d7eed4b29b 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -145,7 +145,17 @@ namespace CNTK } else { - outputDynamicAxes.push_back(Axis::NewUniqueDynamicAxis(L"whereNodeDynamicAxis")); + std::function GetActualSourceVariable; + GetActualSourceVariable = [&GetActualSourceVariable](const Variable& var) -> Variable { + if (var.BlockFunctionVariableMapping() == Variable()) + return var; + else + return GetActualSourceVariable(var.BlockFunctionVariableMapping()); + }; + + auto whereNodeConditionSourceVar = GetActualSourceVariable(inputs[0]); + auto whereNodeSequenceAxis = Axis(std::wstring(L"whereNodeDynamicAxis_conditionVar_") + whereNodeConditionSourceVar.Uid()); + outputDynamicAxes.push_back(whereNodeSequenceAxis); } for (size_t i2 = 1; i2 < inputs[0].DynamicAxes().size(); ++i2) diff --git a/bindings/python/cntk/ops/tests/reshaping_test.py b/bindings/python/cntk/ops/tests/reshaping_test.py index a097a616c63c..f3f799c2e0b6 100644 --- a/bindings/python/cntk/ops/tests/reshaping_test.py +++ b/bindings/python/cntk/ops/tests/reshaping_test.py @@ -325,3 +325,30 @@ def grad_splice(x): unittest_helper(root_op, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision) + + +def test_op_gather_dynamic_axes_equivalence(device_id, precision): + from .. import sequence + + input_data1 = AA([1], dtype=PRECISION_TO_TYPE[precision]) + input_data2 = AA([2], dtype=PRECISION_TO_TYPE[precision]) + + a = I(shape=input_data1.shape, + dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), + name='a') + b = I(shape=input_data2.shape, + dtype=sanitize_dtype_cntk(PRECISION_TO_TYPE[precision]), + name='b') + + is_last_a = sequence.is_last(a) + a_last = sequence.gather(a, is_last_a) + b_last = sequence.gather(b, is_last_a) + z = a_last + b_last + + # create batch + input_data1.shape = (1, 1) + input_data1.shape + input_data2.shape = (1, 1) + input_data2.shape + + res = z.eval({a: input_data1, b: input_data2}) + expected_forward = [[[3.]]] + assert np.array_equal(res, expected_forward) From c72025c1a2caf80d293bdcab1060691070260d3a Mon Sep 17 00:00:00 2001 From: Chris Basoglu Date: Thu, 5 Jan 2017 12:23:20 -0800 Subject: [PATCH 024/120] Move constructor docs to correct spot --- bindings/python/cntk/utils/progress_print.py | 28 +++++++++----------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/bindings/python/cntk/utils/progress_print.py b/bindings/python/cntk/utils/progress_print.py index 91fa4ceb6993..787dff694792 100644 --- a/bindings/python/cntk/utils/progress_print.py +++ b/bindings/python/cntk/utils/progress_print.py @@ -15,23 +15,21 @@ class ProgressPrinter(object): It provides the number of samples, average loss and average metric since the last print or since the start of accumulation. + + Args: + freq (int or None, default None): determines how often + printing will occur. The value of 0 means an geometric + schedule (1,2,4,...). A value > 0 means a arithmetic schedule + (a log print for minibatch number: ``freq``, a log print for minibatch number: 2*``freq``, a log print for minibatch number: 3*``freq``,...), and a value of None means no per-minibatch log. + first (int, default 0): Only start logging after the minibatch number is greater or equal to ``first``. + tag (string, default EmptyString): prepend minibatch log lines with your own string + log_to_file (string or None, default None): if None, output log data to stdout. If a string is passed, the string is path to a file for log data. + distributed_learner (:class:`~cntk.distributed.DistributedLearner` or None, default None): Your learner if you are using distributed parallelism -- each rank's log will go to seperate file. + gen_heartbeat (bool, default False): If True output a progress message every 10 seconds or so to stdout. + num_epochs (int, default 300): The total number of epochs to be trained. Used for some metadata. This parameter is optional. ''' + def __init__(self, freq=None, first=0, tag='', log_to_file=None, distributed_learner=None, gen_heartbeat=False, num_epochs=300): - ''' - Constructor. - - Args: - freq (int or None, default None): determines how often - printing will occur. The value of 0 means an geometric - schedule (1,2,4,...). A value > 0 means a arithmetic schedule - (a log print for minibatch number: ``freq``, a log print for minibatch number: 2*``freq``, a log print for minibatch number: 3*``freq``,...), and a value of None means no per-minibatch log. - first (int, default 0): Only start logging after the minibatch number is greater or equal to ``first``. - tag (string, default EmptyString): prepend minibatch log lines with your own string - log_to_file (string or None, default None): if None, output log data to stdout. If a string is passed, the string is path to a file for log data. - distributed_learner (:class:`~cntk.distributed.DistributedLearner` or None, default None): Your learner if you are using distributed parallelism -- each rank's log will go to seperate file. - gen_heartbeat (bool, default False): If True output a progress message every 10 seconds or so to stdout. - num_epochs (int, default 300): The total number of epochs to be trained. Used for some metadata. This parameter is optional. - ''' from sys import maxsize if freq is None: freq = maxsize From 40d0b79451cd4a6f7f97438945af7dd82e416272 Mon Sep 17 00:00:00 2001 From: KeDengMS Date: Thu, 5 Jan 2017 13:09:12 -0800 Subject: [PATCH 025/120] Add trace for nodes with abs sum to make compare easier, with trace level 1000 --- .../ComputationEnvironment.h | 2 + .../ComputationNetworkEvaluation.cpp | 15 +++-- .../ComputationNetworkLib/ComputationNode.cpp | 65 ++++++++++++------- .../ComputationNetworkLib/ComputationNode.h | 2 +- .../LinearAlgebraNodes.h | 41 ++---------- Source/Math/CPUSparseMatrix.cpp | 2 + Source/Math/GPUSparseMatrix.cu | 7 ++ Source/Math/Matrix.cpp | 4 +- 8 files changed, 70 insertions(+), 68 deletions(-) diff --git a/Source/ComputationNetworkLib/ComputationEnvironment.h b/Source/ComputationNetworkLib/ComputationEnvironment.h index 8d6eb19c0c93..d618e987e8b3 100644 --- a/Source/ComputationNetworkLib/ComputationEnvironment.h +++ b/Source/ComputationNetworkLib/ComputationEnvironment.h @@ -45,6 +45,8 @@ struct ComputationEnvironment // traceLevel int traceLevel = 0; + bool ShouldDumpNode() const { return traceLevel >= 1000; } + // Extreme tracing of node outputs. Make space on your disk. bool IsLogLevelNodeTrace() const { return traceLevel >= 1000000; } diff --git a/Source/ComputationNetworkLib/ComputationNetworkEvaluation.cpp b/Source/ComputationNetworkLib/ComputationNetworkEvaluation.cpp index f28591a73688..cc1c5242d5c3 100644 --- a/Source/ComputationNetworkLib/ComputationNetworkEvaluation.cpp +++ b/Source/ComputationNetworkLib/ComputationNetworkEvaluation.cpp @@ -150,7 +150,7 @@ ComputationNetwork::PARTraversalFlowControlNode::PARTraversalFlowControlNode(con } // Extreme Tracing, part 1/4 - if (node->HasEnvironmentPtr() && node->Environment().IsLogLevelNodeTrace()) + if (node->HasEnvironmentPtr() && node->Environment().ShouldDumpNode()) DumpNode(node, /*dumpGradient=*/false) || DumpNode(node, false); } } @@ -167,7 +167,7 @@ ComputationNetwork::PARTraversalFlowControlNode::PARTraversalFlowControlNode(con node->EndBackprop(); // Extreme Tracing, part 2/4 - if (node->HasEnvironmentPtr() && node->Environment().IsLogLevelNodeTrace() && node->NeedsGradient()) + if (node->HasEnvironmentPtr() && node->Environment().ShouldDumpNode() && node->NeedsGradient()) DumpNode(node, /*dumpGradient=*/true) || DumpNode(node, true); } } @@ -197,12 +197,13 @@ static bool DumpNode(ComputationNodeBasePtr nodep, bool dumpGradient) let dataPtr = dumpGradient ? node->GradientPtr() : node->ValuePtr(); if (!dataPtr) return true; // e.g. SEQ sentinel node - if (dataPtr->GetMatrixType() != MatrixType::DENSE) // for now we can only print dense matrices; since this is for debugging, don't fail just skip - return true; + + bool concise = !(nodep->Environment().IsLogLevelNodeTrace()); + fprintf(stderr, "Dump --> %s%s\n", node->FormatOperationPrototype("").c_str(), dumpGradient ? " Grad" : ""); node->WriteMinibatchWithFormatting(stderr, FrameRange(), SIZE_MAX, SIZE_MAX, false/*transpose*/, /*isCategoryLabel=*/false, /*isSparse=*/false, std::vector(), ""/*sequenceSeparator*/, " "/*sequencePrologue*/, "\n"/*sequenceEpilogue*/, " "/*elementSeparator*/, "\n "/*sampleSeparator*/, - "%13.10f"/*valueFormatString*/, dumpGradient); + "%13.10f"/*valueFormatString*/, dumpGradient, concise); return true; } @@ -257,7 +258,7 @@ static bool DumpNode(ComputationNodeBasePtr nodep, bool dumpGradient) // Extreme Tracing, part 3/4 for (auto& node : m_nestedNodes) { - if (node->HasEnvironmentPtr() && node->Environment().IsLogLevelNodeTrace()) + if (node->HasEnvironmentPtr() && node->Environment().ShouldDumpNode()) { DumpNode(node, /*dumpGradient=*/false) || DumpNode(node, false); } @@ -298,7 +299,7 @@ static bool DumpNode(ComputationNodeBasePtr nodep, bool dumpGradient) // Extreme Tracing, part 4 for (auto& node : m_nestedNodes) { - if (node->HasEnvironmentPtr() && node->Environment().IsLogLevelNodeTrace() && node->NeedsGradient()) + if (node->HasEnvironmentPtr() && node->Environment().ShouldDumpNode() && node->NeedsGradient()) { DumpNode(node, /*dumpGradient=*/true) || DumpNode(node, true); } diff --git a/Source/ComputationNetworkLib/ComputationNode.cpp b/Source/ComputationNetworkLib/ComputationNode.cpp index ef9bd6d36d2b..b0529f9cc93e 100644 --- a/Source/ComputationNetworkLib/ComputationNode.cpp +++ b/Source/ComputationNetworkLib/ComputationNode.cpp @@ -512,7 +512,8 @@ void ComputationNode::WriteMinibatchWithFormatting(FILE* f, const Fram const string& sequencePrologue, const string& sequenceEpilogue, const string& elementSeparator, const string& sampleSeparator, string valueFormatString, - bool outputGradient) const + bool outputGradient, + bool onlyShowAbsSumForDense) const { // get minibatch matrix -> matData, matRows, matStride const Matrix& outputValues = outputGradient ? Gradient() : Value(); @@ -694,34 +695,54 @@ void ComputationNode::WriteMinibatchWithFormatting(FILE* f, const Fram } else { - for (size_t j = 0; j < jend; j++) // loop over output rows --BUGBUG: row index is 'i'!! Rename these!! + if (onlyShowAbsSumForDense) { - if (j > 0) - fprintfOrDie(f, "%s", sampleSep.c_str()); - if (j == jstop && jstop < jend - 1) // if jstop == jend-1 we may as well just print the value instead of '...' + // the concise version to make matrix comparision easier + double absSum = 0; + + #pragma omp parallel for reduction(+:absSum) + for (int i = 0; i < (int)iend; i++) // loop over output rows { - fprintfOrDie(f, "...+%d", (int)(jend - jstop)); // 'nuff said - break; - } - // inject sample tensor index if we are printing row-wise and it's a tensor - if (!transpose && sampleLayout.size() > 1 && !isCategoryLabel) // each row is a different sample dimension - { - for (size_t k = 0; k < sampleLayout.size(); k++) - fprintfOrDie(f, "%c%d", k == 0 ? '[' : ',', (int)((j / sampleLayout.GetStrides()[k])) % sampleLayout[k]); - fprintfOrDie(f, "]\t"); + double absSumLocal = 0; + for (size_t j = 0; j < jend; j++) // loop over elements + { + absSumLocal += abs(seqData[i * istride + j * jstride]); + } + absSum += absSumLocal; } - // print a row of values - for (size_t i = 0; i < iend; i++) // loop over elements + fprintfOrDie(f, "absSum: %f", absSum); + } + else + { + for (size_t j = 0; j < jend; j++) // loop over output rows --BUGBUG: row index is 'i'!! Rename these!! { - if (i > 0) - fprintfOrDie(f, "%s", elementSeparator.c_str()); - if (i == istop && istop < iend - 1) + if (j > 0) + fprintfOrDie(f, "%s", sampleSep.c_str()); + if (j == jstop && jstop < jend - 1) // if jstop == jend-1 we may as well just print the value instead of '...' { - fprintfOrDie(f, "...+%d", (int)(iend - istop)); + fprintfOrDie(f, "...+%d", (int)(jend - jstop)); // 'nuff said break; } - double dval = seqData[i * istride + j * jstride]; - print(dval); + // inject sample tensor index if we are printing row-wise and it's a tensor + if (!transpose && sampleLayout.size() > 1 && !isCategoryLabel) // each row is a different sample dimension + { + for (size_t k = 0; k < sampleLayout.size(); k++) + fprintfOrDie(f, "%c%d", k == 0 ? '[' : ',', (int)((j / sampleLayout.GetStrides()[k])) % sampleLayout[k]); + fprintfOrDie(f, "]\t"); + } + // print a row of values + for (size_t i = 0; i < iend; i++) // loop over elements + { + if (i > 0) + fprintfOrDie(f, "%s", elementSeparator.c_str()); + if (i == istop && istop < iend - 1) + { + fprintfOrDie(f, "...+%d", (int)(iend - istop)); + break; + } + double dval = seqData[i * istride + j * jstride]; + print(dval); + } } } } diff --git a/Source/ComputationNetworkLib/ComputationNode.h b/Source/ComputationNetworkLib/ComputationNode.h index 6e884567e4fc..a2c796828c8d 100644 --- a/Source/ComputationNetworkLib/ComputationNode.h +++ b/Source/ComputationNetworkLib/ComputationNode.h @@ -1865,7 +1865,7 @@ class ComputationNode : public ComputationNodeBase // abstract class that cannot const std::vector& labelMapping, const std::string& sequenceSeparator, const std::string& sequencePrologue, const std::string& sequenceEpilogue, const std::string& elementSeparator, const std::string& sampleSeparator, std::string valueFormatString, - bool outputGradient = false) const; + bool outputGradient = false, bool onlyShowAbsSumForDense = false) const; // simple helper to log the content of a minibatch void DebugLogMinibatch(bool outputGradient = false) const diff --git a/Source/ComputationNetworkLib/LinearAlgebraNodes.h b/Source/ComputationNetworkLib/LinearAlgebraNodes.h index 84d465e87504..f505f6a8e251 100644 --- a/Source/ComputationNetworkLib/LinearAlgebraNodes.h +++ b/Source/ComputationNetworkLib/LinearAlgebraNodes.h @@ -399,24 +399,6 @@ class TimesNodeBase : public ComputationNode, public NumInputs<2> if (inputIndex == 0) // left derivative { - // currently we only support one combination when the input is sparse - // If input data is sparse, then gradient is block sparse. - if (InputRef(1).Value().GetMatrixType() == SPARSE && InputRef(0).Gradient().GetMatrixType() == DENSE && Gradient().GetMatrixType() == DENSE) - { - // We need a sparse matrix for the gradient. We allocate a new one instead of switching the type in place - // since switching in place may affect other nodes who share this matrix due to memory sharing - auto& currentInput0GradientMatrixRef = InputRef(0).Gradient(); - auto newInput0SparseGradientMatrix = std::make_shared>(currentInput0GradientMatrixRef.GetNumRows(), - currentInput0GradientMatrixRef.GetNumCols(), - currentInput0GradientMatrixRef.GetPreferredDeviceId(), - SPARSE, - MatrixFormat::matrixFormatSparseBlockCol); - - // BUGBUG: Copy over the current contents since we accumulate into the gradient matrix instead of overwriting the content - // newInput0SparseGradientMatrix.AssignValuesOf(currentInput0GradientMatrixRef); - InputRef(0).GradientPtrRef() = newInput0SparseGradientMatrix; - } - auto input0Gradient = OneSampleTensorFor(0, /*gradient=*/true, fr.AllowBroadcast()); auto input1 = OneSampleTensorFor(1, /*gradient=*/false, fr.AllowBroadcast()); auto outputGradient = OneSampleTensorFor(-1, /*gradient=*/true, fr); @@ -427,7 +409,6 @@ class TimesNodeBase : public ComputationNode, public NumInputs<2> } else if (inputIndex == 1) // right derivative { - // BUGBUG: Above block produces case of sparse second input a sparse gradient gor first input. We should either have corresponding code here or not at all. auto input0 = OneSampleTensorFor(0, /*gradient=*/false, fr.AllowBroadcast()); auto input1Gradient = OneSampleTensorFor(1, /*gradient=*/true, fr.AllowBroadcast()); auto outputGradient = OneSampleTensorFor(-1, /*gradient=*/true, fr); @@ -572,23 +553,11 @@ class TimesNodeBase : public ComputationNode, public NumInputs<2> // this is a special handling case. We need to allocate sparse matrix directly instead of from pool. if (Input(0)->NeedsGradient() && Input(1)->Value().GetMatrixType() == SPARSE) { - Input(0)->CreateGradientMatrixIfNull(); - - // We need a sparse matrix for the gradient. We allocate a new one instead of switching the type in place - // since switching in place may affect other nodes who share this matrix due to memory sharing - auto& currentInput0GradientMatrixRef = InputRef(0).Gradient(); - if (currentInput0GradientMatrixRef.GetMatrixType() != SPARSE) - { - auto newInput0SparseGradientMatrix = std::make_shared>(currentInput0GradientMatrixRef.GetNumRows(), - currentInput0GradientMatrixRef.GetNumCols(), - currentInput0GradientMatrixRef.GetPreferredDeviceId(), - SPARSE, - MatrixFormat::matrixFormatSparseBlockCol); - - // BUGBUG: Copy over the current contents since we accumulate into the gradient matrix instead of overwriting the content - // newInput0SparseGradientMatrix.AssignValuesOf(currentInput0GradientMatrixRef); - InputRef(0).GradientPtrRef() = newInput0SparseGradientMatrix; - } + InputRef(0).GradientPtrRef() = std::make_shared>(0, // size would be initialized later + 0, + Gradient().GetPreferredDeviceId(), + SPARSE, + MatrixFormat::matrixFormatSparseBlockCol); } // we need to call base allocation at end since we will need to allocate special ones first diff --git a/Source/Math/CPUSparseMatrix.cpp b/Source/Math/CPUSparseMatrix.cpp index 151727f5ea66..d64945d06aba 100644 --- a/Source/Math/CPUSparseMatrix.cpp +++ b/Source/Math/CPUSparseMatrix.cpp @@ -1589,6 +1589,7 @@ template CPUSparseMatrix CPUSparseMatrix::ColumnSlice(size_t startCo template CPUMatrix CPUSparseMatrix::CopyColumnSliceToDense(size_t startColumn, size_t numCols) const; template void CPUSparseMatrix::AssignColumnSliceToDense(CPUMatrix&, size_t startColumn, size_t numCols) const; template CPUSparseMatrix& CPUSparseMatrix::operator=(const CPUSparseMatrix& deepCopyFrom); +template void CPUSparseMatrix::ScaleAndAdd(char, class Microsoft::MSR::CNTK::CPUSparseMatrix const &, class Microsoft::MSR::CNTK::CPUMatrix &); // Support template CPUSparseMatrix::CPUSparseMatrix(const MatrixFormat format, const size_t numRows, const size_t numCols, const size_t size); @@ -1611,6 +1612,7 @@ template CPUSparseMatrix CPUSparseMatrix::ColumnSlice(size_t start template CPUMatrix CPUSparseMatrix::CopyColumnSliceToDense(size_t startColumn, size_t numCols) const; template void CPUSparseMatrix::AssignColumnSliceToDense(CPUMatrix&, size_t startColumn, size_t numCols) const; template CPUSparseMatrix& CPUSparseMatrix::operator=(const CPUSparseMatrix& deepCopyFrom); +template void CPUSparseMatrix::ScaleAndAdd(short, class Microsoft::MSR::CNTK::CPUSparseMatrix const &, class Microsoft::MSR::CNTK::CPUMatrix &); template CPUSparseMatrix::CPUSparseMatrix(const MatrixFormat, const size_t, const size_t, const size_t); template CPUSparseMatrix::~CPUSparseMatrix(); diff --git a/Source/Math/GPUSparseMatrix.cu b/Source/Math/GPUSparseMatrix.cu index 4d760aa0e6ff..b353103f5f32 100644 --- a/Source/Math/GPUSparseMatrix.cu +++ b/Source/Math/GPUSparseMatrix.cu @@ -291,6 +291,11 @@ void GPUSparseMatrix::CopyToDenseMatrix(GPUMatrix& denseMatr CUSPARSE_CALL(cusparseDcsc2dense(cusparseHandle, int(GetNumRows()), int(GetNumCols()), descr, (double*) Buffer(), RowLocation(), ColLocation(), (double*) denseMatrix.Data(), int(GetNumRows()))); } } + else if (GetFormat() == MatrixFormat::matrixFormatSparseBlockCol || GetFormat() == MatrixFormat::matrixFormatSparseBlockRow) + { + denseMatrix.SetValue((ElemType)0); + ScaleAndAdd(1, *this, denseMatrix); + } else { NOT_IMPLEMENTED; @@ -2727,6 +2732,7 @@ template void GPUSparseMatrix::SetValue(CPUSparseMatrix const&); template void GPUSparseMatrix::SetValue(GPUSparseMatrix const&); template void GPUSparseMatrix::SetValue(GPUMatrix const&); //template void GPUSparseMatrix::SetValue(CPUMatrix const&); +template GPUMatrix GPUSparseMatrix::CopyToDenseMatrix() const; template void GPUSparseMatrix::CopyToDenseMatrix(GPUMatrix&) const; template void GPUSparseMatrix::CopyToCPUSparseMatrix(CPUSparseMatrix&) const; template void GPUSparseMatrix::ChangeDeviceTo(int); @@ -2750,6 +2756,7 @@ template void GPUSparseMatrix::SetValue(CPUSparseMatrix const&); template void GPUSparseMatrix::SetValue(GPUSparseMatrix const&); template void GPUSparseMatrix::SetValue(GPUMatrix const&); //template void GPUSparseMatrix::SetValue(CPUMatrix const&); +template GPUMatrix GPUSparseMatrix::CopyToDenseMatrix() const; template void GPUSparseMatrix::CopyToDenseMatrix(GPUMatrix&) const; template void GPUSparseMatrix::CopyToCPUSparseMatrix(CPUSparseMatrix&) const; template void GPUSparseMatrix::ChangeDeviceTo(int); diff --git a/Source/Math/Matrix.cpp b/Source/Math/Matrix.cpp index 7b2ac2560c19..2edf6f046984 100644 --- a/Source/Math/Matrix.cpp +++ b/Source/Math/Matrix.cpp @@ -717,8 +717,8 @@ ElemType* Matrix::CopyToArray() const nullptr, return m_CPUMatrix->CopyToArray(), return m_GPUMatrix->CopyToArray(), - NOT_IMPLEMENTED, - NOT_IMPLEMENTED); + { CPUMatrix tmpDense(m_CPUSparseMatrix->GetNumRows(), m_CPUSparseMatrix->GetNumCols()); tmpDense.SetValue((ElemType)0); CPUSparseMatrix::ScaleAndAdd((ElemType)1, *m_CPUSparseMatrix, tmpDense); return tmpDense.CopyToArray(); }, + return m_GPUSparseMatrix->CopyToDenseMatrix().CopyToArray()); } //memory will be allocated by the callee if not enough but need to be deleted by the caller after it's done From b7f5beb0ab4aa2cf14bbd4916f96ad74148e7d9d Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Thu, 5 Jan 2017 16:38:15 -0800 Subject: [PATCH 026/120] Support for MaxUnpooling --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 21 +++++++ Source/CNTKv2LibraryDll/BackCompat.cpp | 12 ++++ Source/CNTKv2LibraryDll/CompositeFunction.cpp | 14 +++++ Source/CNTKv2LibraryDll/Function.cpp | 22 +++++++ Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 29 +++++++++ Source/CNTKv2LibraryDll/PrimitiveFunction.h | 3 + Source/CNTKv2LibraryDll/PrimitiveOpType.h | 1 + .../ConvolutionalNodes.h | 6 +- Source/Math/ConvolutionEngine.h | 1 + bindings/python/cntk/ops/__init__.py | 46 ++++++++++++++ bindings/python/cntk/ops/tests/kernel_test.py | 60 +++++++++++++++++-- 11 files changed, 209 insertions(+), 6 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 87e74b7b1c32..b7bb758916b7 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -3024,6 +3024,27 @@ namespace CNTK const NDShape& upperPad = {0}, const std::wstring& name = L""); + /// + /// Unpooling types + /// + enum class UnpoolingType + { + Max + }; + + /// + /// Create an instance of the CNTK built-in Unpooling operation on specified tensor input operands with the specified type and shape + /// + CNTK_API FunctionPtr Unpooling(const Variable& operand, + const Variable& poolingInput, + UnpoolingType UnpoolingType, + const NDShape& UnpoolingWindowShape, + const NDShape& strides = { 1 }, + const std::vector& autoPadding = { false }, + const NDShape& lowerPad = { 0 }, + const NDShape& upperPad = { 0 }, + const std::wstring& name = L""); + /// /// TODO: /// diff --git a/Source/CNTKv2LibraryDll/BackCompat.cpp b/Source/CNTKv2LibraryDll/BackCompat.cpp index 81f152ca2e26..52541bda0502 100644 --- a/Source/CNTKv2LibraryDll/BackCompat.cpp +++ b/Source/CNTKv2LibraryDll/BackCompat.cpp @@ -344,6 +344,18 @@ namespace CNTK opType = PrimitiveOpType::ROIPooling; } + else if (node->OperationName() == OperationNameOf(MaxUnpoolingNode)) + { + auto unpoolingNode = node->As>(); + primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameUnpoolingType] = (size_t)UnpoolingType::Max; + primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameUnpoolingWindowShape] = AsNDShape(unpoolingNode->KernelShape()); + primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameStrides] = AsNDShape(unpoolingNode->Strides()); + primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameAutoPadding] = AsDictionaryValueVector(unpoolingNode->AutoPad()); + primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameLowerPad] = AsNDShape(unpoolingNode->LowerPad()); + primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameUpperPad] = AsNDShape(unpoolingNode->UpperPad()); + + opType = PrimitiveOpType::Unpooling; + } else if (node->OperationName() == OperationNameOf(PoolingNode)) { auto poolingNode = node->As>(); diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.cpp b/Source/CNTKv2LibraryDll/CompositeFunction.cpp index 0719fac1512a..0232ae75b8bd 100644 --- a/Source/CNTKv2LibraryDll/CompositeFunction.cpp +++ b/Source/CNTKv2LibraryDll/CompositeFunction.cpp @@ -577,6 +577,20 @@ namespace CNTK computationNodePtr = New>(network->GetDeviceId(), internalNodeName, AsCNTKPoolKind(poolingType), AsTensorShape(poolingWindowsShape), AsTensorShape(strides), autoPadding, AsTensorShape(lowerPad), AsTensorShape(upperPad), ImageLayoutKind::CHW); break; } + case PrimitiveOpType::Unpooling: + { + UnpoolingType unpoolingType = (UnpoolingType)(functionConfig[PrimitiveFunction::AttributeNameUnpoolingType].Value()); + auto unpoolingWindowShape = functionConfig[PrimitiveFunction::AttributeNameUnpoolingWindowShape].Value(); + auto strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); + auto lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); + auto upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); + auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); + if (unpoolingType == UnpoolingType::Max) + computationNodePtr = New>(network->GetDeviceId(), internalNodeName, AsTensorShape(unpoolingWindowShape), AsTensorShape(strides), autoPadding, AsTensorShape(lowerPad), AsTensorShape(upperPad), ImageLayoutKind::CHW); + else + LogicError("Unknown unpooling type"); + break; + } case PrimitiveOpType::SumAll: computationNodePtr = New>(network->GetDeviceId(), internalNodeName); break; diff --git a/Source/CNTKv2LibraryDll/Function.cpp b/Source/CNTKv2LibraryDll/Function.cpp index f053d265bdb8..7d2ee36ca191 100644 --- a/Source/CNTKv2LibraryDll/Function.cpp +++ b/Source/CNTKv2LibraryDll/Function.cpp @@ -1062,6 +1062,28 @@ namespace CNTK return UnaryOp(PrimitiveOpType::Pooling, operand, std::move(additionalProperties), name); } + FunctionPtr Unpooling(const Variable& operand, + const Variable& poolingInput, + UnpoolingType unpoolingType, + const NDShape& poolingWindowShape, + const NDShape& strides, + const std::vector& autoPadding, + const NDShape& lowerPad, + const NDShape& upperPad, + const std::wstring& name) + { + auto additionalProperties = Dictionary(); + additionalProperties[PrimitiveFunction::AttributeNameUnpoolingType] = (size_t)unpoolingType; + additionalProperties[PrimitiveFunction::AttributeNameUnpoolingWindowShape] = poolingWindowShape; + additionalProperties[PrimitiveFunction::AttributeNameStrides] = strides; + additionalProperties[PrimitiveFunction::AttributeNameAutoPadding] = AsDictionaryValueVector(autoPadding); + additionalProperties[PrimitiveFunction::AttributeNameLowerPad] = lowerPad; + additionalProperties[PrimitiveFunction::AttributeNameUpperPad] = upperPad; + + std::vector operands = { operand, poolingInput}; + return CompositeFunction::Create(MakeSharedObject(PrimitiveOpType::Unpooling, operands, std::move(additionalProperties), name), name); + } + FunctionPtr BatchNormalization(const Variable& operand, const Variable& scale, const Variable& bias, diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index fc74dd3bffbc..f0237ea9eb51 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -71,6 +71,8 @@ namespace CNTK /*static*/ const std::wstring PrimitiveFunction::AttributeNameRecurrentOp = L"recurrentOp"; /*static*/ const std::wstring PrimitiveFunction::AttributeNameRngSeed = L"rngSeed"; /*static*/ const std::wstring PrimitiveFunction::AttributeNameRngOffset = L"rngOffset"; + /*static*/ const std::wstring PrimitiveFunction::AttributeNameUnpoolingType = L"unpoolingType"; + /*static*/ const std::wstring PrimitiveFunction::AttributeNameUnpoolingWindowShape = L"unpoolingWindowShape"; /*static*/ std::vector PrimitiveFunction::GetOutputVariables(PrimitiveOpType op, std::vector& inputs, @@ -334,6 +336,33 @@ namespace CNTK outputShape = ConvolutionOpOutputShape(op, inputShape, poolingWindowsShape, outputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, false, inferDimensions); break; } + case PrimitiveOpType::Unpooling: + { + assert(inputs.size() == 2); + + auto inputShape = inputs[0].Shape(); + auto poolInputShape = inputs[1].Shape(); + outputShape = poolInputShape; + + // Finding the shape of an unpooling operation is ambiguous + // For example a 4x4x1 input with a 5x5 kernel a stride of 2x2 + // and padding could have resulted from pooling a 7x7 or 8x8 image + // Therefore what needs to happen here is to check whether the + // outputShape can be pooled into the inputShape using the specified attributes + auto unpoolingWindowShape = functionConfig[PrimitiveFunction::AttributeNameUnpoolingWindowShape].Value(); + auto strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); + auto lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); + auto upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); + auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); + NDShape inputMapCount = { 1 }; + std::vector sharing = { true }; + + NDShape inferredInputShape = ConvolutionOpOutputShape(PrimitiveOpType::Pooling, outputShape, unpoolingWindowShape, inputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, false, inferDimensions); + if (inferredInputShape != inputShape) + RuntimeError("The shape of the unpooling operand is different from the result of pooling the poolingInput argument using the provided options"); + + break; + } case PrimitiveOpType::SumAll: assert(inputs.size() == 1); outputShape = {1}; diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.h b/Source/CNTKv2LibraryDll/PrimitiveFunction.h index 121dd669e32e..198654f630d5 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.h +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.h @@ -86,6 +86,7 @@ namespace CNTK {PrimitiveOpType::Cos, L"Cos"}, {PrimitiveOpType::Pass, L"Pass"}, { PrimitiveOpType::Block, L"Block" }, + { PrimitiveOpType::Unpooling, L"Unpooling" }, }; inline const std::wstring& PrimitiveOpTypeName(PrimitiveOpType opType) @@ -217,6 +218,8 @@ namespace CNTK static const std::wstring AttributeNameNumLayers; static const std::wstring AttributeNameHiddenSize; static const std::wstring AttributeNameRecurrentOp; + static const std::wstring AttributeNameUnpoolingType; + static const std::wstring AttributeNameUnpoolingWindowShape; protected: PrimitiveFunction(const std::vector& blockInputs, const std::vector& blockOutputs, Dictionary&& functionConfig, const std::wstring& functionName, const std::wstring& uid) diff --git a/Source/CNTKv2LibraryDll/PrimitiveOpType.h b/Source/CNTKv2LibraryDll/PrimitiveOpType.h index 13385f9c000b..4fd3e0cec330 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveOpType.h +++ b/Source/CNTKv2LibraryDll/PrimitiveOpType.h @@ -67,6 +67,7 @@ namespace CNTK Cos = 55, Pass = 56, Block = 57, + Unpooling = 58, // New op types should only be appended to the end of this list. // If you append here, also add checks in SerializationTests (CheckEnumValuesNotModified) // and bump up PrimitiveFunction::s_serializationVersion diff --git a/Source/ComputationNetworkLib/ConvolutionalNodes.h b/Source/ComputationNetworkLib/ConvolutionalNodes.h index 2ea05fcd1347..6b2e1a3af611 100644 --- a/Source/ComputationNetworkLib/ConvolutionalNodes.h +++ b/Source/ComputationNetworkLib/ConvolutionalNodes.h @@ -915,8 +915,12 @@ class MaxUnpoolingNode : public ConvolutionNodeBase, public NumInputs< // Same as in case of deconvolution, node input (inputShape) is really the output of the max pooling // and node output (outDims) is pooling input. - auto outputShape = ConvolveGeometry::ComputeInputShape(inputShape, m_kernelShape, m_mapCount, m_stride, + auto outputShape = GetInputSampleLayout(1); + auto inferredShape = ConvolveGeometry::ComputeOutputShape(outputShape, m_kernelShape, m_mapCount, m_stride, m_sharing, m_autoPad, m_lowerPad, m_upperPad); + if (inputShape != inferredShape) + InvalidArgument("The shape of the unpooling operand is different from the result of pooling the poolingInput argument using the provided options"); + SetDims(outputShape, HasMBLayout()); if (isFinalValidationPass) { diff --git a/Source/Math/ConvolutionEngine.h b/Source/Math/ConvolutionEngine.h index b6cd1150a716..c996e2ffa8ab 100644 --- a/Source/Math/ConvolutionEngine.h +++ b/Source/Math/ConvolutionEngine.h @@ -33,6 +33,7 @@ enum class PoolKind Average }; + #pragma warning(push) #pragma warning(disable : 4251) diff --git a/bindings/python/cntk/ops/__init__.py b/bindings/python/cntk/ops/__init__.py index 2ff8416255b0..17692873b4e6 100644 --- a/bindings/python/cntk/ops/__init__.py +++ b/bindings/python/cntk/ops/__init__.py @@ -433,6 +433,52 @@ def pooling(operand, pooling_type, pooling_window_shape, strides=(1,), auto_padd lower_pad, upper_pad, name) +from cntk.cntk_py import UnpoolingType_Max +MAX_UNPOOLING = UnpoolingType_Max +@typemap +def unpooling(operand, pooling_input, unpooling_type, unpooling_window_shape, strides=(1,), auto_padding=[False], + lower_pad=(0,), upper_pad=(0,), name=''): + ''' + The pooling operations compute a new tensor by selecting the maximum or average value in the pooling input. + In the case of average pooling with padding, the average is only over the valid region. + + N-dimensional pooling allows to create max or average pooling of any dimensions, stride or padding. + + Example: + >>> img = np.reshape(np.arange(16, dtype = np.float32), [1, 4, 4]) + >>> x = C.input_variable(img.shape) + >>> C.pooling(x, C.AVG_POOLING, (2,2), (2,2)).eval({x : [img]}) + array([[[[[ 2.5, 4.5], + [ 10.5, 12.5]]]]], dtype=float32) + >>> C.pooling(x, C.MAX_POOLING, (2,2), (2,2)).eval({x : [img]}) + array([[[[[ 5., 7.], + [ 13., 15.]]]]], dtype=float32) + + Args: + operand: unpooling input + pooling_input: input to the corresponding pooling node + unpooling_type: only :const:`~cntk.ops.MAX_UNPOOLING` is supported now + unpooling_window_shape: dimensions of the unpooling window + strides (default 1): strides. + auto_padding: automatic padding flags for each input dimension. + lower_pad: precise lower padding for each input dimension + upper_pad: precise upper padding for each input dimension + name (str, optional): the name of the Function instance in the network + Returns: + :class:`~cntk.ops.functions.Function` + ''' + from cntk.cntk_py import unpooling + operand = sanitize_input(operand) + pooling_input = sanitize_input(pooling_input) + unpooling_window_shape = sanitize_shape(unpooling_window_shape) + strides = sanitize_shape(strides) + lower_pad = sanitize_shape(lower_pad) + upper_pad = sanitize_shape(upper_pad) + return unpooling(operand, pooling_input, unpooling_type, + unpooling_window_shape, strides, auto_padding, + lower_pad, upper_pad, name) + + @typemap def batch_normalization(operand, scale, bias, running_mean, running_inv_std, spatial, normalization_time_constant=5000, blend_time_constant=0, diff --git a/bindings/python/cntk/ops/tests/kernel_test.py b/bindings/python/cntk/ops/tests/kernel_test.py index 9f8e1a23854d..f214b036d813 100644 --- a/bindings/python/cntk/ops/tests/kernel_test.py +++ b/bindings/python/cntk/ops/tests/kernel_test.py @@ -11,7 +11,7 @@ import numpy as np import pytest from .ops_test_utils import unittest_helper, _test_unary_op, AA, I, precision, PRECISION_TO_TYPE, constant, cntk_device -from cntk.ops import AVG_POOLING, MAX_POOLING +from cntk.ops import AVG_POOLING, MAX_POOLING, MAX_UNPOOLING from ...utils import sanitize_dtype_cntk CONVOLUTION_OPERANDS = [ @@ -153,14 +153,16 @@ def test_op_avg_pooling(input_size, pooling_window, strides, result, device_id, ([1, 2, 2, 4, 3], # input_size (2, 2, 1), # pooling_window (2, 2, 1), # strides + [False], # autopad [[[[ 16., 17., 18.], [ 22., 23., 24.]]], [[[ 40., 41., 42.], [ 46., 47., 48.]]]]), # result - ([1, 2, 4, 4 ,4], + ([1, 2, 4, 4, 4], (2, 2, 2), (2, 2, 2), + [False], [[[[ 22., 24.], [ 30., 32.]], [[ 54., 56.], @@ -169,11 +171,20 @@ def test_op_avg_pooling(input_size, pooling_window, strides, result, device_id, [ 94., 96.]], [[ 118., 120.], [ 126., 128.]]]]), + + ([1, 1, 1, 8, 8], + (5, 5), + (2, 2), + [True], + [[[[ 19., 21., 23., 24.], + [ 35., 37., 39., 40.], + [ 51., 53., 55., 56.], + [ 59., 61., 63., 64.]]]]) ] -@pytest.mark.parametrize("input_size, pooling_window, strides, result", MAX_POOLING_DATA) -def test_op_max_pooling(input_size, pooling_window, strides, result, device_id, precision): +@pytest.mark.parametrize("input_size, pooling_window, strides, autopad, result", MAX_POOLING_DATA) +def test_op_max_pooling(input_size, pooling_window, strides, autopad, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] # fill input operand with a sequence 1,2,3,... til total size and then @@ -196,7 +207,7 @@ def test_op_max_pooling(input_size, pooling_window, strides, result, device_id, backward += np.asarray(input_operand == element) from cntk import pooling - input_op = pooling(a, MAX_POOLING, pooling_window, strides) + input_op = pooling(a, MAX_POOLING, pooling_window, strides, autopad) forward_input = {a: input_operand} @@ -207,6 +218,45 @@ def test_op_max_pooling(input_size, pooling_window, strides, result, device_id, forward_input, expected_forward, expected_backward, device_id=device_id, precision=precision) + +@pytest.mark.parametrize("input_size, pooling_window, strides, autopad, result", MAX_POOLING_DATA) +def test_op_max_unpooling(input_size, pooling_window, strides, autopad, result, device_id, precision): + dt = PRECISION_TO_TYPE[precision] + + # fill input operand with a sequence 1,2,3,... til total size and then + # resize to input_size + total_size = np.prod(input_size) + x = np.arange(1, total_size + 1, 1, dtype=dt) + input_operand = x.reshape(input_size) + + a = I(shape=input_operand.shape[2:], + dtype=sanitize_dtype_cntk(precision), + needs_gradient=True, + name='a') + + pooling_result = np.asarray(result, dtype=dt) + max_elements = pooling_result.reshape(pooling_result.size).tolist() + + # place 1.0s where maximum elements are + backward = np.zeros_like(input_operand) + for element in max_elements: + backward += np.asarray(input_operand == element) + + from cntk import pooling, unpooling + p = pooling(a, MAX_POOLING, pooling_window, strides, autopad) + u = unpooling(p, a, MAX_UNPOOLING, pooling_window, strides, autopad) + q = pooling(u, MAX_POOLING, pooling_window, strides, autopad) + + forward_input = {a: input_operand} + + expected_forward = backward * input_operand + expected_backward = {a: backward} + + unittest_helper(u, + forward_input, expected_forward, expected_backward, + device_id=device_id, precision=precision) + assert np.allclose(p.eval(forward_input), q.eval(forward_input)) + # ROI pooling test setup # --- forward --- # input convFeatureMap 3x3 map, values [[1,2,3][4,5,6][7,8,9]] From 303faaa2c3573d897ba56f81f24dd484de38a368 Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Thu, 5 Jan 2017 18:05:56 -0800 Subject: [PATCH 027/120] actual unpooling usage --- Source/Math/ConvolutionEngine.h | 1 - bindings/python/cntk/ops/__init__.py | 26 +++++++++++++------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/Source/Math/ConvolutionEngine.h b/Source/Math/ConvolutionEngine.h index c996e2ffa8ab..b6cd1150a716 100644 --- a/Source/Math/ConvolutionEngine.h +++ b/Source/Math/ConvolutionEngine.h @@ -33,7 +33,6 @@ enum class PoolKind Average }; - #pragma warning(push) #pragma warning(disable : 4251) diff --git a/bindings/python/cntk/ops/__init__.py b/bindings/python/cntk/ops/__init__.py index 17692873b4e6..32f148854efa 100644 --- a/bindings/python/cntk/ops/__init__.py +++ b/bindings/python/cntk/ops/__init__.py @@ -73,7 +73,7 @@ def as_block(composite, block_arguments_map, block_op_name, block_instance_name= Args: composite: The composite Function that the block encapsulates - block_arguments_map: A list of tuples, mapping from block's underlying composite's arguments to + block_arguments_map: A list of tuples, mapping from block's underlying composite's arguments to actual variables they are connected to block_op_name: Name of the op that the block represents block_instance_name (str, optional): the name of the block Function in the network @@ -439,20 +439,20 @@ def pooling(operand, pooling_type, pooling_window_shape, strides=(1,), auto_padd def unpooling(operand, pooling_input, unpooling_type, unpooling_window_shape, strides=(1,), auto_padding=[False], lower_pad=(0,), upper_pad=(0,), name=''): ''' - The pooling operations compute a new tensor by selecting the maximum or average value in the pooling input. - In the case of average pooling with padding, the average is only over the valid region. - - N-dimensional pooling allows to create max or average pooling of any dimensions, stride or padding. + Unpools the ``operand`` using information from ``pooling_input``. Unpooling mirrors the operations + performed by pooling and depends on the values provided to the corresponding pooling node. The output + should have the same shape as pooling_input. Pooling the result of an unpooling operation should + give back the original input. Example: >>> img = np.reshape(np.arange(16, dtype = np.float32), [1, 4, 4]) >>> x = C.input_variable(img.shape) - >>> C.pooling(x, C.AVG_POOLING, (2,2), (2,2)).eval({x : [img]}) - array([[[[[ 2.5, 4.5], - [ 10.5, 12.5]]]]], dtype=float32) - >>> C.pooling(x, C.MAX_POOLING, (2,2), (2,2)).eval({x : [img]}) - array([[[[[ 5., 7.], - [ 13., 15.]]]]], dtype=float32) + >>> y = C.pooling(x, C.MAX_POOLING, (2,2), (2,2)) + >>> C.unpooling(y, x, C.MAX_UNPOOLING, (2,2), (2,2)).eval({x : [img]}) + array([[[[[ 0., 0., 0., 0.], + [ 0., 5., 0., 7.], + [ 0., 0., 0., 0.], + [ 0., 13., 0., 15.]]]]], dtype=float32) Args: operand: unpooling input @@ -474,7 +474,7 @@ def unpooling(operand, pooling_input, unpooling_type, unpooling_window_shape, st strides = sanitize_shape(strides) lower_pad = sanitize_shape(lower_pad) upper_pad = sanitize_shape(upper_pad) - return unpooling(operand, pooling_input, unpooling_type, + return unpooling(operand, pooling_input, unpooling_type, unpooling_window_shape, strides, auto_padding, lower_pad, upper_pad, name) @@ -1596,7 +1596,7 @@ def optimized_rnnstack(operand, weights, hidden_size, num_layers, Returns: :class:`~cntk.ops.functions.Function` ''' - # FIXME figure out how to only SKIP the doctest in CPU + # FIXME figure out how to only SKIP the doctest in CPU from cntk.cntk_py import optimized_rnnstack operand = sanitize_input(operand) if recurrent_op not in set(['lstm','gru','relu','tanh']): From a026a272bf0e03f8db1c311182f2cb8b675d8767 Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Thu, 5 Jan 2017 18:24:32 -0800 Subject: [PATCH 028/120] addressed cr feedback --- Tests/UnitTests/MathTests/MatrixTests.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Tests/UnitTests/MathTests/MatrixTests.cpp b/Tests/UnitTests/MathTests/MatrixTests.cpp index eab7e41bd1d4..75f4fbfc84d3 100644 --- a/Tests/UnitTests/MathTests/MatrixTests.cpp +++ b/Tests/UnitTests/MathTests/MatrixTests.cpp @@ -772,7 +772,7 @@ BOOST_FIXTURE_TEST_CASE(MatrixAssignXOf, RandomSeedFixture) c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); - // Check for self-assignment 1 + // Check for self-assignment (c = c + b) auto tolerance = 5e-5; c.AssignSumOf(c, b); foreach_coord (i, j, c) @@ -783,7 +783,7 @@ BOOST_FIXTURE_TEST_CASE(MatrixAssignXOf, RandomSeedFixture) b.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); - // Check for self-assignment 2 + // Check for self-assignment (c = b + c) c.AssignSumOf(b, c); foreach_coord (i, j, c) { @@ -793,7 +793,7 @@ BOOST_FIXTURE_TEST_CASE(MatrixAssignXOf, RandomSeedFixture) b.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); - // Check for self-assignment 3 + // Check for self-assignment (c = c + a .* c) c.AssignSumOf(a, b); c.AddElementProductOf(a, c); foreach_coord(i, j, c) @@ -804,7 +804,7 @@ BOOST_FIXTURE_TEST_CASE(MatrixAssignXOf, RandomSeedFixture) b.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); - // Check for self-assignment 4 + // Check for self-assignment (c = c + c .* a) c.AssignSumOf(a, b); c.AddElementProductOf(c, a); foreach_coord(i, j, c) @@ -815,7 +815,7 @@ BOOST_FIXTURE_TEST_CASE(MatrixAssignXOf, RandomSeedFixture) b.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); c.TransferToDeviceIfNotThere(c_deviceIdZero, true, false, true); - // Check for self-assignment 5 + // Check for self-assignment (c = c + c .* c) c.AssignSumOf(a, b); c.AddElementProductOf(c, c); foreach_coord(i, j, c) From c185789c64fc6ef7027a640b7a98220f816aaa58 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Mon, 12 Dec 2016 18:48:14 +0100 Subject: [PATCH 029/120] implement copyto complete copyto for dense --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 10 ++ Source/CNTKv2LibraryDll/Value.cpp | 199 +++++++++++++++++++++- 2 files changed, 207 insertions(+), 2 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 64c91d298b86..e8b5d8ac7fec 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -943,6 +943,16 @@ namespace CNTK /// virtual void CopyFrom(const Value& source); + /// + /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. + /// The sequence buffer is on CPU. + /// The Value should have the same axes as variable. + /// + template + CNTK_API void CopyTo(const NDShape& sampleShape, std::vector>& outputData); + + CNTK_API void CopyTo(const NDShape& sampleShape, std::vector>& outputData); + private: template static void AppendSparseSequenceData(const NDArrayViewPtr& sequenceData, std::vector& colStarts, std::vector& rowIndices, std::vector& nonZeroValues, size_t maxSequenceLength); diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index 598c8d3a0eba..3896c14139be 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -286,7 +286,7 @@ namespace CNTK deviceValueData = valueData; } else - deviceValueData = valueData->DeepClone(device, readOnly); + deviceValueData = valueData->DeepClone(device, readOnly); return MakeSharedObject(deviceValueData, deviceValueMask); } @@ -305,7 +305,7 @@ namespace CNTK InvalidArgument("Value::Create:: The number of elements in the vector containing sequence data must be a multiple of the size of specified sampel shape"); auto sequenceLength = currentSequence.size() / numElementsPerSample; - auto sequenceDataShape = sampleShape.AppendShape({ sequenceLength }); + auto sequenceDataShape = sampleShape.AppendShape({sequenceLength}); sequencesData.push_back(MakeSharedObject(sequenceDataShape, currentSequence)); } @@ -359,6 +359,199 @@ namespace CNTK } } + // If outputData.size() is 0, CNTK will alocate stroage for data. Otherwise, the caller is reposible for allocating sufficient stroage space for saving the data. + template + void Value::CopyTo(const NDShape& sampleShape, std::vector>& outputData) + { + // Check the data type matches + if (AsDataType() != GetDataType()) + InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); + + // Todo: convert sparse into dense. + if (GetStorageFormat() != StorageFormat::Dense) + InvalidArgument("Only the dense storage format is supported now."); + + auto valueRank = Shape().Rank(); + if ((valueRank <= 2) || (sampleShape != Shape().SubShape(0, valueRank - 2)) + RuntimeError("The variable and the Value does not have the same tensor shape."); + + // Copy data to the CPU device if required. + NDArrayViewPtr cpuArrayView; + NDMaskPtr cpuNDMask; + if (Device() != DeviceDescriptor::CPUDevice()) + { + cpuArrayView = m_data->DeepClone(DeviceDescriptor::CPUDevice()); + cpuNDMask = m_mask->DeepClone(DeviceDescriptor::CPUDevice()); + } + else + { + cpuArrayView = m_data; + cpuNDMask = m_mask; + } + + auto maskData = cpuNDMask->DataBuffer(); + auto valueData = cpuArrayView->DataBuffer(); + auto numOfSequences = Shape()[valueRank - 1]; + auto maxSequenceLen = Shape()[valueRank - 2]; + auto sampleSize = sampleShape.TotalSize(); + bool needStorage = false; + + if (outputData.size() == 0) + { + needStorage = true; + } + else if (numOfSequences > outputData.size()) + { + RuntimeError("The size of the output buffer is smaller than the number of sequences."); + } + + const ElementType *first, *last; + ElementType *dest; + std::vector seqBuf; + size_t count, current; + for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) + { + size_t seqStart = seqIndex * maxSequenceLen; + + // Check the number of valid elements. + // Not using MaskedCount() to avoid extra data copy. + count = 0; + for (int i = 0; i < maxSequenceLen; i++) + { + if (maskData[seqStart + i] != MaskKind::Invalid) + count++; + } + + if (needStorage) + { + auto p = new std::vector(count * sampleSize); + outputData.push_back(*p); + } + + seqBuf = outputData[seqIndex]; + if (count * sampleSize > seqBuf.size()) + { + RuntimeError("The sequenth %lu contains more data than the size of the provided vector.\n", (unsigned long)seqIndex); + } + + dest = seqBuf.data(); + current = 0; + while (current < maxSequenceLen) + { + // find first valid mask. + while ((maskData[seqStart + current] == MaskKind::Invalid) && (current < maxSequenceLen)) + current++; + first = valueData + (seqStart + current) * sampleSize; + + // find the next invalid mask. + while ((maskData[seqStart + current] != MaskKind::Invalid) && (current < maxSequenceLen)) + current++; + last = valueData + (seqStart + current) * sampleSize; + + if (last > first) + { + std::copy(first, last, dest); + dest += last - first; + assert(dest <= seqBuf.data() + count); + } + } + assert(dest == seqBuf.data() + count); + } + } + + void Value::CopyTo(const NDShape& sampleShape, std::vector>& outputData) + { + + if (sampleShape.Rank() != 1) + RuntimeError("Only data of 1-D tensor can be copied to OneHot vector."); + + auto valueRank = Shape().Rank(); + // Check the shape matches. + if (sampleShape != Shape().SubShape(0, valueRank - 2)) + InvalidArgument("The variable and the value does not have the same tensor shape."); + + //// Todo: convert sparse into dense. + //if (GetStorageFormat() != StorageFormat::Dense) + // InvalidArgument("Only the dense storage format is supported now."); + + //// Copy data to the CPU device if required. + //NDArrayViewPtr cpuArrayView; + //NDMaskPtr cpuNDMask; + //if (Device != DeviceDescriptor::CPUDevice()) + //{ + // cpuArrayView = m_data->DeepClone(DeviceDescriptor::CPUDevice()); + // cpuNDMask = m_mask->DeepClone(DeviceDescriptor::CPUDevice()); + //} + //else + //{ + // cpuArrayView = m_data; + // cpuNDMask = m_mask; + //} + + //auto maskData = cpuNDMask->DataBuffer(); + //auto valueData = cpuArrayView->DataBuffer(); + //auto numOfSequences = Shape[valueRank - 1]; + //auto maxSequenceLen = Shape[valueRank - 2]; + //auto sampleSize = variable.Shape().TotalSize(); + + //if (outputData == nullptr) + //{ + // outputData = new std::vector>(numOfSequences); + //} + //if (numOfSequences > outputData.size()) + //{ + // RuntimeError("The size of output buffer is smaller than the number of sequences."); + //} + + //ElementType *first, *last, *dest; + //size_t count; + //for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) + //{ + // size_t seqStart = seqIndex * maxSequenceLen; + + // // Check the number of valid elements. + // // Not using MaskedCount() to avoid extra data copy. + // count = 0; + // for (int i = 0; i < maxSequenceLen; i++) + // { + // if (maskData[seqStart + i] != MaskKind::Invalid) + // count++; + // } + + // auto seqBuf = outputData[seqIndex]; + // if (seqBuf == nullptr) + // { + // outputData[seqIndex] = seqBuf = new std::vector(count * sampleSize); + // } + // if (count * sampleSize > seqBuf.size()) + // { + // RuntimeError("The sequenth %lu contains more data than the buffer size.\n", (unsigned long)seqIndex); + // } + // dest = seqBuf; + + // while (current < maxSequenceLen) + // { + // // find first valid mask. + // while ((maskData[seqStart + current] == MaskKind::Invalid) && (current < maxSequenceLen)) + // current++; + // first = valueData + (seqStart + current) * sampleSize; + + // // find the next invalid mask. + // while ((maskData[seqStart + current] != MaskKind::Invalid) && (current < maxSequenceLen)) + // current++; + // last = valueData + (seqStart + current) * sampleSize; + + // if (last > first) + // { + // std::copy(first, last, dest); + // dest += last - first; + // assert(dest <= seqBuf + count); + // } + // } + // assert(dest == seqBuf + count); + //} + } + void PackedValue::Unpack() const { if (m_packedDataLayout && (m_packedDataLayout->GetNumTimeSteps() != 1) && (m_packedDataLayout->GetNumSequences() != 1) && Internal::IsAutomaticUnpackingOfPackedValuesDisabled()) @@ -397,4 +590,6 @@ namespace CNTK template /*static*/ CNTK_API ValuePtr Value::Create(const NDShape& sampleShape, const std::vector>& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); + template CNTK_API void Value::CopyTo(const NDShape& sampleShape, std::vector>& sequences); + template CNTK_API void Value::CopyTo(const NDShape& sampleShape, std::vector>& sequences); } From f5fdbeb028969a2facab0d46246489aa9d185308 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Sat, 17 Dec 2016 06:34:10 +0100 Subject: [PATCH 030/120] refactor Value::CopyTo to allocate memory in .h file; adapt unit tests; add sequencesLens as parameter for Value::CopyTo --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 130 +++++++++++++++- Tests/UnitTests/V2LibraryTests/Main.cpp | 10 +- Tests/UnitTests/V2LibraryTests/ValueTests.cpp | 143 ++++++++++++++++++ 3 files changed, 276 insertions(+), 7 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index e8b5d8ac7fec..d8377a269101 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -949,9 +949,70 @@ namespace CNTK /// The Value should have the same axes as variable. /// template - CNTK_API void CopyTo(const NDShape& sampleShape, std::vector>& outputData); + void CopyTo(const NDShape& sampleShape, std::vector>& sequences) + { + std::vector seqLens; + CopyTo(sampleShape, sequences, seqLens, true); + } + + template + void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLens, bool isResizeable = true) + { + // Check the data type matches + if (AsDataType() != GetDataType()) + InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); + + // Todo: convert sparse into dense. + if (GetStorageFormat() != StorageFormat::Dense) + InvalidArgument("Only the dense storage format is supported now."); + + auto valueRank = Shape().Rank(); + auto sampleRank = sampleShape.Rank(); + if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != Shape().SubShape(0, sampleRank))) + RuntimeError("The variable and the Value does not have the same tensor shape."); + + CheckAndResizeOutputBuffer(sampleRank, sampleShape.TotalSize(), sequences, sequenceLens, isResizeable); + + CopyToImpl(sampleShape, sequences, sequenceLens); + } + + /// + /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. + /// The sequence buffer is on CPU. + /// The Value should have the same axes as variable. + /// + template + void CopyTo(const NDShape& sampleShape, std::vector>& sequences) + { + std::vector seqLens; + CopyTo(sampleShape, sequences, seqLens, true); + } + + template + void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLens, bool isResizeable = true) + { + // Check the data type matches + if (AsDataType() != GetDataType()) + InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); + + // Todo: convert sparse into dense. + if (GetStorageFormat() != StorageFormat::Dense) + InvalidArgument("Only the dense storage format is supported now."); + + if (sampleShape[0] != sampleShape.TotalSize()) + InvalidArgument("") + + auto valueRank = Shape().Rank(); + auto sampleRank = sampleShape.Rank(); + if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != Shape().SubShape(0, sampleRank))) + RuntimeError("The variable and the Value does not have the same tensor shape."); + + // For OneHot vector, only 1 value is needed for a sample. + CheckAndResizeOutputBuffer(sampleRank, 1, sequences, sequenceLens, isResizeable); + + // CopyToImpl(sampleShape, sequences, sequenceLens); + } - CNTK_API void CopyTo(const NDShape& sampleShape, std::vector>& outputData); private: template @@ -959,6 +1020,71 @@ namespace CNTK CNTK_API static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly, bool createNewCopy); + template + CNTK_API void CopyToImpl(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLens); + + template + void CheckAndResizeOutputBuffer(const size_t sampleRank, const size_t sampleSize, std::vector>& sequences, std::vector& sequenceLens, bool isResizeable) + { + auto valueRank = Shape().Rank(); + size_t numOfSequences; + size_t maxSequenceLen; + + if (valueRank == sampleRank + 1) + { + // no batch axis, only sequence axis + numOfSequences = 1; + maxSequenceLen = Shape()[valueRank - 1]; + } + else + { + assert(valueRank == sampleRank + 2); + numOfSequences = Shape()[valueRank - 1]; + maxSequenceLen = Shape()[valueRank - 2]; + } + + // resize the sequnce length buffer to reflect the number of sequences in output. + if (sequenceLens.size() < numOfSequences) + sequenceLens.resize(numOfSequences); + + // Check whether the additional space in the sequences output buffer needs to be allocated if it is resizeable. + if (isResizeable) + { + const MaskKind* maskData = nullptr; + if (m_mask != nullptr) + maskData = Device() != DeviceDescriptor::CPUDevice() ? m_mask->DeepClone(DeviceDescriptor::CPUDevice())->DataBuffer() : m_mask->DataBuffer(); + + size_t sampleCount, seqStart; + for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) + { + if (maskData == nullptr) + { + sampleCount = maxSequenceLen; + } + else + { + seqStart = seqIndex * maxSequenceLen; + sampleCount = 0; + for (size_t i = 0; i < maxSequenceLen; i++) + if (maskData[seqStart + i] != MaskKind::Invalid) + sampleCount++; + } + + if (seqIndex < sequences.size()) + { + if (sequences[seqIndex].size() < sampleCount * sampleSize) + sequences[seqIndex].resize(sampleCount * sampleSize); + } + else + { + assert(seqIndex == sequences.size()); + auto p = new std::vector(sampleCount * sampleSize); + sequences.push_back(*p); + } + } + } + } + // Disallow copy and move construction and assignment Value(const Value&) = delete; Value& operator=(const Value&) = delete; Value(Value&&) = delete; Value& operator=(Value&&) = delete; diff --git a/Tests/UnitTests/V2LibraryTests/Main.cpp b/Tests/UnitTests/V2LibraryTests/Main.cpp index d6ffa5703020..e1e376f5cc7b 100644 --- a/Tests/UnitTests/V2LibraryTests/Main.cpp +++ b/Tests/UnitTests/V2LibraryTests/Main.cpp @@ -46,13 +46,13 @@ int main() // Lets disable automatic unpacking of PackedValue object to detect any accidental unpacking // which will have a silent performance degradation otherwise - Internal::SetAutomaticUnpackingOfPackedValues(/*disable =*/ true); + //Internal::SetAutomaticUnpackingOfPackedValues(/*disable =*/ true); - // Note: Run the device selection tests first since later tests - // may interfere with device selection by freezing default device - DeviceSelectionTests(); + //// Note: Run the device selection tests first since later tests + //// may interfere with device selection by freezing default device + //DeviceSelectionTests(); - NDArrayViewTests(); + //NDArrayViewTests(); ValueTests(); TensorTests(); FunctionTests(); diff --git a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp index 74569285931a..dc2e44020c35 100644 --- a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp @@ -239,6 +239,141 @@ void ValueCreationOneHotWithNDMaskTest(const DeviceDescriptor device, bool readO CheckValue(testValue, {vocabSize, maxSeqLen, numberOfSequences}, vocabSize, data, seqLenList); } +template +void CheckCopyToOutput(const size_t sampleSize, std::vector> expected, std::vector> actual) +{ + std::vector seqLens(0); + CheckCopyToOutput(sampleSize, expected, actual, seqLens); +} + +template +void CheckCopyToOutput(const size_t sampleSize, std::vector>& expected, std::vector>& actual, std::vector& seqLens) +{ + bool useSeqLens; + if (seqLens.size() != 0) + { + useSeqLens = true; + if (seqLens.size() < expected.size()) + ReportFailure("The seqLens size does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), seqLens.size()); + else + { + for (size_t i = expected.size(); i < seqLens.size(); i++) + if (seqLens[i] != 0) + ReportFailure("The seqLens contains invalid data."); + } + + if (actual.size() < expected.size()) + ReportFailure("The number of sequences does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), actual.size()); + } + else + { + useSeqLens = false; + if (expected.size() != actual.size()) + ReportFailure("The number of sequences does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), actual.size()); + } + + for (size_t i = 0; i < expected.size(); i++) + { + auto len = useSeqLens ? seqLens[i] * sampleSize : actual[i].size(); + if ((actual[i].size() < len) || (expected[i].size() != len)) + { + ReportFailure("Seq " PRIu64 " does not match.\n", i); + } + for (size_t j = 0; j < len; j++) + { + if (expected[i][j] != actual[i][j]) + { + ReportFailure("Seq " PRIu64 " does not match.\n", i); + } + } + } +} + +template +void ValueCopyToDenseTest(const DeviceDescriptor device) +{ + NDShape sampleShape{{2, 3}}; + auto sampleSize = sampleShape.TotalSize(); + std::vector> output; + std::vector> sequences; + std::vector seqLens; + + // Check single sample. + sequences.clear(); + sequences.resize(1, std::vector(sampleSize)); + // Todo: use generateSequences in common.h + for (size_t i = 0; i < sampleSize; i++) + sequences[0][i] = static_cast(i); + auto val = Value::Create(sampleShape, sequences, device); + val->CopyTo(sampleShape, output); + CheckCopyToOutput(sampleSize, sequences, output); + + // Check batch of sample. + size_t batchCount = 2; + sequences.clear(); + sequences.resize(2, std::vector(sampleSize)); + for (size_t s = 0; s < batchCount; s++) + { + for (size_t i = 0; i < sampleSize; i++) + { + sequences[s][i] = static_cast(s * 10 + i); + } + } + val = Value::Create(sampleShape, sequences, device); + VerifyException([&val, &sampleShape, &output, &seqLens]() { + val->CopyTo(sampleShape, output, seqLens, false); + }, "The output buffer is too small."); + val->CopyTo(sampleShape, output, seqLens); + CheckCopyToOutput(sampleSize, sequences, output, seqLens); + + // Check sequence of sample + size_t sampleCount = 4; + sequences.clear(); + sequences.resize(1, std::vector(sampleSize * sampleCount)); + for (size_t i = 0; i < sampleSize * sampleCount; i++) + { + sequences[0][i] = static_cast(i); + } + val = Value::Create(sampleShape, sequences, device); + VerifyException([&val, &sampleShape, &output, &seqLens]() { + val->CopyTo(sampleShape, output, seqLens, false); + }, "The output buffer is too small."); + val->CopyTo(sampleShape, output, seqLens); + CheckCopyToOutput(sampleSize, sequences, output, seqLens); + + // Check batch of sequence of the same length, no mask needed. + batchCount = 4; + sampleCount = 3; + sequences.clear(); + sequences.resize(batchCount, std::vector(sampleSize * sampleCount)); + for (size_t s = 0; s < batchCount; s++) + { + for (size_t i = 0; i < sampleSize * sampleCount; i++) + { + sequences[s][i] = static_cast(s * 10 + i); + } + } + val = Value::Create(sampleShape, sequences, device); + val->CopyTo(sampleShape, output, seqLens); + CheckCopyToOutput(sampleSize, sequences, output, seqLens); + + // Check batch of sequecnes with different length, mask needed. + std::vector sampleCountList{6, 9, 2, 1, 5, 3, 4}; + batchCount = sampleCountList.size(); + sequences.clear(); + sequences.resize(batchCount, std::vector(0)); + for (size_t s = 0; s < batchCount; s++) + { + for (size_t i = 0; i < sampleSize * sampleCountList[s]; i++) + { + sequences[s].push_back(static_cast(s * 10 + i)); + } + } + val = Value::Create(sampleShape, sequences, device); + val->CopyTo(sampleShape, output, seqLens); + CheckCopyToOutput(sampleSize, sequences, output, seqLens); +} + void TestSettingParameterValuesManually(const DeviceDescriptor& device) { auto v1_1 = MakeSharedObject(0.5, NDShape({ 2, 2 }), device); @@ -312,8 +447,12 @@ void ValueTests() ValueCreationOneHotNoNDMaskTest(DeviceDescriptor::CPUDevice(), true); ValueCreationOneHotWithNDMaskTest(DeviceDescriptor::CPUDevice(), false); ValueCreationOneHotWithNDMaskTest(DeviceDescriptor::CPUDevice(), true); +<<<<<<< HEAD SparseSequenceBatchValueCreationTest(300, 7, DeviceDescriptor::CPUDevice()); SparseSequenceBatchValueCreationTest(2300, 1, DeviceDescriptor::CPUDevice()); +======= + ValueCopyToDenseTest(DeviceDescriptor::CPUDevice()); +>>>>>>> 0a57cbc... refactor Value::CopyTo to allocate memory in .h file; adapt unit tests; add sequencesLens as parameter for Value::CopyTo if (IsGPUAvailable()) { @@ -327,7 +466,11 @@ void ValueTests() ValueCreationOneHotNoNDMaskTest(DeviceDescriptor::GPUDevice(0), true); ValueCreationOneHotWithNDMaskTest(DeviceDescriptor::GPUDevice(0), false); ValueCreationOneHotWithNDMaskTest(DeviceDescriptor::GPUDevice(0), true); +<<<<<<< HEAD SparseSequenceBatchValueCreationTest(50000, 1, DeviceDescriptor::GPUDevice(0)); SparseSequenceBatchValueCreationTest(6000, 6, DeviceDescriptor::GPUDevice(0)); +======= + ValueCopyToDenseTest(DeviceDescriptor::GPUDevice(0)); +>>>>>>> 0a57cbc... refactor Value::CopyTo to allocate memory in .h file; adapt unit tests; add sequencesLens as parameter for Value::CopyTo } } From f1cbc08e9431da2fe6ba784111c62b0ad724c015 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Sat, 17 Dec 2016 22:07:11 +0100 Subject: [PATCH 031/120] add CopyTo for onehot vector; refactor code to share between CopyTo methods; fix bug using GPU device. The cpuArrayView must be alive in the function scope add CopyTo for OneHot; support Value in both dense and sparse format --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 77 ++-- Source/CNTKv2LibraryDll/Value.cpp | 331 +++++++++--------- Tests/UnitTests/V2LibraryTests/ValueTests.cpp | 313 +++++++++++++---- 3 files changed, 442 insertions(+), 279 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index d8377a269101..3d65d3e195b4 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -21,6 +21,7 @@ #include #include #include +#include #ifdef SWIG #define final @@ -956,24 +957,10 @@ namespace CNTK } template - void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLens, bool isResizeable = true) + void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable = true) { - // Check the data type matches - if (AsDataType() != GetDataType()) - InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); - - // Todo: convert sparse into dense. - if (GetStorageFormat() != StorageFormat::Dense) - InvalidArgument("Only the dense storage format is supported now."); - - auto valueRank = Shape().Rank(); - auto sampleRank = sampleShape.Rank(); - if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != Shape().SubShape(0, sampleRank))) - RuntimeError("The variable and the Value does not have the same tensor shape."); - - CheckAndResizeOutputBuffer(sampleRank, sampleShape.TotalSize(), sequences, sequenceLens, isResizeable); - - CopyToImpl(sampleShape, sequences, sequenceLens); + CheckAndResizeOutputBuffer(sampleShape.Rank(), sampleShape.TotalSize(), sequences, sequenceLengths, isResizeable); + CopyToVector(sampleShape, sequences, sequenceLengths); } /// @@ -981,39 +968,27 @@ namespace CNTK /// The sequence buffer is on CPU. /// The Value should have the same axes as variable. /// - template void CopyTo(const NDShape& sampleShape, std::vector>& sequences) { std::vector seqLens; CopyTo(sampleShape, sequences, seqLens, true); } - template - void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLens, bool isResizeable = true) + void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable = true) { - // Check the data type matches - if (AsDataType() != GetDataType()) - InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); - - // Todo: convert sparse into dense. - if (GetStorageFormat() != StorageFormat::Dense) - InvalidArgument("Only the dense storage format is supported now."); - - if (sampleShape[0] != sampleShape.TotalSize()) - InvalidArgument("") - - auto valueRank = Shape().Rank(); - auto sampleRank = sampleShape.Rank(); - if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != Shape().SubShape(0, sampleRank))) - RuntimeError("The variable and the Value does not have the same tensor shape."); - // For OneHot vector, only 1 value is needed for a sample. - CheckAndResizeOutputBuffer(sampleRank, 1, sequences, sequenceLens, isResizeable); - - // CopyToImpl(sampleShape, sequences, sequenceLens); + CheckAndResizeOutputBuffer(sampleShape.Rank(), 1, sequences, sequenceLengths, isResizeable); + auto dataType = GetDataType(); + if (dataType == DataType::Float) + { + CopyToVector(sampleShape, sequences, sequenceLengths); + } + else if (dataType == DataType::Double) + { + CopyToVector(sampleShape, sequences, sequenceLengths); + } } - private: template static void AppendSparseSequenceData(const NDArrayViewPtr& sequenceData, std::vector& colStarts, std::vector& rowIndices, std::vector& nonZeroValues, size_t maxSequenceLength); @@ -1021,10 +996,16 @@ namespace CNTK CNTK_API static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly, bool createNewCopy); template - CNTK_API void CopyToImpl(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLens); + CNTK_API void CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); template - void CheckAndResizeOutputBuffer(const size_t sampleRank, const size_t sampleSize, std::vector>& sequences, std::vector& sequenceLens, bool isResizeable) + CNTK_API void CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); + + template + void CopyToImpl(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); + + template + void CheckAndResizeOutputBuffer(const size_t sampleRank, const size_t sampleSize, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable) { auto valueRank = Shape().Rank(); size_t numOfSequences; @@ -1044,15 +1025,19 @@ namespace CNTK } // resize the sequnce length buffer to reflect the number of sequences in output. - if (sequenceLens.size() < numOfSequences) - sequenceLens.resize(numOfSequences); + if (sequenceLengths.size() < numOfSequences) + sequenceLengths.resize(numOfSequences); // Check whether the additional space in the sequences output buffer needs to be allocated if it is resizeable. if (isResizeable) { const MaskKind* maskData = nullptr; - if (m_mask != nullptr) - maskData = Device() != DeviceDescriptor::CPUDevice() ? m_mask->DeepClone(DeviceDescriptor::CPUDevice())->DataBuffer() : m_mask->DataBuffer(); + NDMaskPtr cpuMask = nullptr; + if (Mask() != nullptr) + { + cpuMask = (Device() != DeviceDescriptor::CPUDevice()) ? Mask()->DeepClone(DeviceDescriptor::CPUDevice()) : Mask(); + maskData = cpuMask->DataBuffer(); + } size_t sampleCount, seqStart; for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index 3896c14139be..02197cfb30c8 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -359,199 +359,152 @@ namespace CNTK } } - // If outputData.size() is 0, CNTK will alocate stroage for data. Otherwise, the caller is reposible for allocating sufficient stroage space for saving the data. + template + void DirectCopy(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest, size_t& destSampleStart); + + template + void CopyDenseToOneHot(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest, size_t& destSampleStart); + template - void Value::CopyTo(const NDShape& sampleShape, std::vector>& outputData) - { + void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths) + { // Check the data type matches if (AsDataType() != GetDataType()) InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); - // Todo: convert sparse into dense. - if (GetStorageFormat() != StorageFormat::Dense) - InvalidArgument("Only the dense storage format is supported now."); + CopyToImpl(sampleShape, sequences, sequenceLengths); + } + + template + CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths) + { + if (sampleShape[0] != sampleShape.TotalSize()) + InvalidArgument(""); + + CopyToImpl(sampleShape, sequences, sequenceLengths); + } + template + void Value::CopyToImpl(const NDShape& sampleShape, + std::vector>& sequences, + std::vector& sequenceLengths) + { auto valueRank = Shape().Rank(); - if ((valueRank <= 2) || (sampleShape != Shape().SubShape(0, valueRank - 2)) - RuntimeError("The variable and the Value does not have the same tensor shape."); + auto sampleRank = sampleShape.Rank(); + if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != Shape().SubShape(0, sampleRank))) + RuntimeError("The variable and the Value does not have the same tensor shape."); - // Copy data to the CPU device if required. - NDArrayViewPtr cpuArrayView; - NDMaskPtr cpuNDMask; - if (Device() != DeviceDescriptor::CPUDevice()) + size_t numOfSequences; + size_t maxSequenceLen; + if (valueRank == sampleShape.Rank() + 1) { - cpuArrayView = m_data->DeepClone(DeviceDescriptor::CPUDevice()); - cpuNDMask = m_mask->DeepClone(DeviceDescriptor::CPUDevice()); + // no batch axis, only sequence axis + numOfSequences = 1; + maxSequenceLen = Shape()[valueRank - 1]; } else { - cpuArrayView = m_data; - cpuNDMask = m_mask; + assert(valueRank == sampleShape.Rank() + 2); + numOfSequences = Shape()[valueRank - 1]; + maxSequenceLen = Shape()[valueRank - 2]; } - auto maskData = cpuNDMask->DataBuffer(); - auto valueData = cpuArrayView->DataBuffer(); - auto numOfSequences = Shape()[valueRank - 1]; - auto maxSequenceLen = Shape()[valueRank - 2]; - auto sampleSize = sampleShape.TotalSize(); - bool needStorage = false; + // Check output buffer size + if (sequences.size() < numOfSequences) + RuntimeError("The size of output buffer is too small"); - if (outputData.size() == 0) + // Check sequenceLengths size. + if (sequenceLengths.size() < numOfSequences) { - needStorage = true; + RuntimeError("The size of sequenceLengths does not match."); } - else if (numOfSequences > outputData.size()) + else { - RuntimeError("The size of the output buffer is smaller than the number of sequences."); + for (size_t i = numOfSequences; i < sequenceLengths.size(); i++) + sequenceLengths[i] = 0; } - const ElementType *first, *last; - ElementType *dest; - std::vector seqBuf; - size_t count, current; - for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) + // Copy data to the CPU device if required. + const ValueType *valueData; + const MaskKind* maskData; + NDArrayViewPtr cpuArrayView; + NDMaskPtr cpuMask; + if (Device() != DeviceDescriptor::CPUDevice()) { - size_t seqStart = seqIndex * maxSequenceLen; - - // Check the number of valid elements. - // Not using MaskedCount() to avoid extra data copy. - count = 0; - for (int i = 0; i < maxSequenceLen; i++) + // Todo: leverage sparse if the original NDArrayView is in spase. + cpuArrayView = MakeSharedObject(GetDataType(), Data()->Shape(), DeviceDescriptor::CPUDevice()); + cpuArrayView->CopyFrom(*Data()); + cpuMask = Mask() != nullptr ? Mask()->DeepClone(DeviceDescriptor::CPUDevice()) : nullptr; + } + else + { + // Todo: direct process sparse data without copy + if (GetStorageFormat() != StorageFormat::Dense) { - if (maskData[seqStart + i] != MaskKind::Invalid) - count++; + cpuArrayView = MakeSharedObject(GetDataType(), Data()->Shape(), DeviceDescriptor::CPUDevice()); + cpuArrayView->CopyFrom(*Data()); } - - if (needStorage) + else { - auto p = new std::vector(count * sampleSize); - outputData.push_back(*p); + cpuArrayView = Data(); } + cpuMask = Mask(); + } + valueData = cpuArrayView->DataBuffer(); + maskData = cpuMask != nullptr ? cpuMask->DataBuffer() : nullptr; - seqBuf = outputData[seqIndex]; - if (count * sampleSize > seqBuf.size()) + auto sampleSize = sampleShape.TotalSize(); + for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) + { + size_t seqStart = seqIndex * maxSequenceLen; + size_t destSampleCount = 0; + if (maskData == nullptr) { - RuntimeError("The sequenth %lu contains more data than the size of the provided vector.\n", (unsigned long)seqIndex); + // Todo: if function pointer or lambda could support template, switch to use them. + if (typeid(DestType) == typeid(size_t)) + { + CopyDenseToOneHot(valueData + seqStart * sampleSize, maxSequenceLen, sampleSize, sequences[seqIndex], destSampleCount); + } + else + { + DirectCopy(valueData + seqStart * sampleSize, maxSequenceLen, sampleSize, sequences[seqIndex], destSampleCount); + } + sequenceLengths[seqIndex] = destSampleCount; } - - dest = seqBuf.data(); - current = 0; - while (current < maxSequenceLen) + else { - // find first valid mask. - while ((maskData[seqStart + current] == MaskKind::Invalid) && (current < maxSequenceLen)) - current++; - first = valueData + (seqStart + current) * sampleSize; - - // find the next invalid mask. - while ((maskData[seqStart + current] != MaskKind::Invalid) && (current < maxSequenceLen)) - current++; - last = valueData + (seqStart + current) * sampleSize; - - if (last > first) + // NDMask is not null + size_t current = seqStart; + size_t seqEnd = seqStart + maxSequenceLen; + while (current < seqEnd) { - std::copy(first, last, dest); - dest += last - first; - assert(dest <= seqBuf.data() + count); + // find first valid mask. + while ((current < seqEnd) && (maskData[current] == MaskKind::Invalid)) + current++; + auto sampleStart = current; + + // find the next invalid mask. + while ((current < seqEnd) && (maskData[current] != MaskKind::Invalid)) + current++; + assert(current >= sampleStart); + if (current > sampleStart) + { + // Todo: if function pointer or lambda could support template, switch to use them. + if (typeid(DestType) == typeid(size_t)) + { + CopyDenseToOneHot(valueData + seqStart * sampleSize, current - sampleStart, sampleSize, sequences[seqIndex], destSampleCount); + } + else + { + DirectCopy(valueData + seqStart * sampleSize, current - sampleStart, sampleSize, sequences[seqIndex], destSampleCount); + } + } } + sequenceLengths[seqIndex] = destSampleCount; } - assert(dest == seqBuf.data() + count); } } - void Value::CopyTo(const NDShape& sampleShape, std::vector>& outputData) - { - - if (sampleShape.Rank() != 1) - RuntimeError("Only data of 1-D tensor can be copied to OneHot vector."); - - auto valueRank = Shape().Rank(); - // Check the shape matches. - if (sampleShape != Shape().SubShape(0, valueRank - 2)) - InvalidArgument("The variable and the value does not have the same tensor shape."); - - //// Todo: convert sparse into dense. - //if (GetStorageFormat() != StorageFormat::Dense) - // InvalidArgument("Only the dense storage format is supported now."); - - //// Copy data to the CPU device if required. - //NDArrayViewPtr cpuArrayView; - //NDMaskPtr cpuNDMask; - //if (Device != DeviceDescriptor::CPUDevice()) - //{ - // cpuArrayView = m_data->DeepClone(DeviceDescriptor::CPUDevice()); - // cpuNDMask = m_mask->DeepClone(DeviceDescriptor::CPUDevice()); - //} - //else - //{ - // cpuArrayView = m_data; - // cpuNDMask = m_mask; - //} - - //auto maskData = cpuNDMask->DataBuffer(); - //auto valueData = cpuArrayView->DataBuffer(); - //auto numOfSequences = Shape[valueRank - 1]; - //auto maxSequenceLen = Shape[valueRank - 2]; - //auto sampleSize = variable.Shape().TotalSize(); - - //if (outputData == nullptr) - //{ - // outputData = new std::vector>(numOfSequences); - //} - //if (numOfSequences > outputData.size()) - //{ - // RuntimeError("The size of output buffer is smaller than the number of sequences."); - //} - - //ElementType *first, *last, *dest; - //size_t count; - //for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) - //{ - // size_t seqStart = seqIndex * maxSequenceLen; - - // // Check the number of valid elements. - // // Not using MaskedCount() to avoid extra data copy. - // count = 0; - // for (int i = 0; i < maxSequenceLen; i++) - // { - // if (maskData[seqStart + i] != MaskKind::Invalid) - // count++; - // } - - // auto seqBuf = outputData[seqIndex]; - // if (seqBuf == nullptr) - // { - // outputData[seqIndex] = seqBuf = new std::vector(count * sampleSize); - // } - // if (count * sampleSize > seqBuf.size()) - // { - // RuntimeError("The sequenth %lu contains more data than the buffer size.\n", (unsigned long)seqIndex); - // } - // dest = seqBuf; - - // while (current < maxSequenceLen) - // { - // // find first valid mask. - // while ((maskData[seqStart + current] == MaskKind::Invalid) && (current < maxSequenceLen)) - // current++; - // first = valueData + (seqStart + current) * sampleSize; - - // // find the next invalid mask. - // while ((maskData[seqStart + current] != MaskKind::Invalid) && (current < maxSequenceLen)) - // current++; - // last = valueData + (seqStart + current) * sampleSize; - - // if (last > first) - // { - // std::copy(first, last, dest); - // dest += last - first; - // assert(dest <= seqBuf + count); - // } - // } - // assert(dest == seqBuf + count); - //} - } - void PackedValue::Unpack() const { if (m_packedDataLayout && (m_packedDataLayout->GetNumTimeSteps() != 1) && (m_packedDataLayout->GetNumSequences() != 1) && Internal::IsAutomaticUnpackingOfPackedValuesDisabled()) @@ -585,11 +538,69 @@ namespace CNTK } } + template + void DirectCopy(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest, size_t& destSampleStart) + { + if (typeid(ElementType) != typeid(DestType)) + RuntimeError("Source and destination must be the same data type."); + + DestType *destData = dest.data(); + if ((destSampleStart + sampleCount) * sampleSize > dest.size()) + RuntimeError("The output buffer is too small."); + std::copy(source, source + sampleCount * sampleSize, reinterpret_cast(destData + destSampleStart * sampleSize)); + destSampleStart += sampleCount; + } + + template + void CopyDenseToOneHot(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest, size_t& destSampleStart) + { + if (typeid(DestType) != typeid(size_t)) + { + RuntimeError("The destination data type must be size_t."); + } + + const ElementType *currentp = source; + const ElementType *lastp = source + sampleCount * sampleSize; + while (currentp < lastp) + { + auto sampleEndp = currentp + sampleSize; + auto indexp = std::find_if(currentp, sampleEndp, [](const ElementType val) { + return val != 0; + }); + + if (indexp == sampleEndp) + { + RuntimeError("Cannot convert to onehot vector: the sample does not have any non-zero value."); + } + else + { + if (std::find_if(indexp + 1, sampleEndp, [](const ElementType val) { + return val != 0; + }) != sampleEndp) + { + RuntimeError("Cannot convert to onehot vector: more than one non-zero value in the sample."); + } + else + { + if (destSampleStart >= dest.size()) + RuntimeError("The output buffer is too small."); + else + { + dest[destSampleStart++] = static_cast(indexp - currentp); + } + } + } + currentp += sampleSize; + } + assert(currentp == lastp); + } + // Explicit template instantiations template /*static*/ CNTK_API ValuePtr Value::Create(const NDShape& sampleShape, const std::vector>& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(const NDShape& sampleShape, const std::vector>& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); - template CNTK_API void Value::CopyTo(const NDShape& sampleShape, std::vector>& sequences); - template CNTK_API void Value::CopyTo(const NDShape& sampleShape, std::vector>& sequences); + template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequencesLens); + template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequencesLens); + } diff --git a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp index dc2e44020c35..8066e16a6fe3 100644 --- a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp @@ -147,6 +147,7 @@ void FillDenseMatrixData(vector>& databuf, const vector void ValueCreationNoNDMaskTest(const DeviceDescriptor device, bool readOnly) { + //Todo: test numberOfSequences == 1: no batch access, need to adapt checkShape(), CheckValue() size_t numberOfSequences = 5; size_t seqLen = 4; vector dims{3, 2}; @@ -242,24 +243,24 @@ void ValueCreationOneHotWithNDMaskTest(const DeviceDescriptor device, bool readO template void CheckCopyToOutput(const size_t sampleSize, std::vector> expected, std::vector> actual) { - std::vector seqLens(0); - CheckCopyToOutput(sampleSize, expected, actual, seqLens); + std::vector actualSeqLens(0); + CheckCopyToOutput(sampleSize, expected, actual, actualSeqLens); } template -void CheckCopyToOutput(const size_t sampleSize, std::vector>& expected, std::vector>& actual, std::vector& seqLens) +void CheckCopyToOutput(const size_t sampleSize, std::vector>& expected, std::vector>& actual, std::vector& actualSeqLens) { bool useSeqLens; - if (seqLens.size() != 0) + if (actualSeqLens.size() != 0) { useSeqLens = true; - if (seqLens.size() < expected.size()) - ReportFailure("The seqLens size does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), seqLens.size()); + if (actualSeqLens.size() < expected.size()) + ReportFailure("The actualSeqLens size does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), actualSeqLens.size()); else { - for (size_t i = expected.size(); i < seqLens.size(); i++) - if (seqLens[i] != 0) - ReportFailure("The seqLens contains invalid data."); + for (size_t i = expected.size(); i < actualSeqLens.size(); i++) + if (actualSeqLens[i] != 0) + ReportFailure("The actualSeqLens contains invalid data."); } if (actual.size() < expected.size()) @@ -274,7 +275,7 @@ void CheckCopyToOutput(const size_t sampleSize, std::vector> input; std::vector> output; - std::vector> sequences; - std::vector seqLens; + std::vector expectedSeqLens; + std::vector actualSeqLens; + //Todo: add tests sparse to dense. // Check single sample. - sequences.clear(); - sequences.resize(1, std::vector(sampleSize)); - // Todo: use generateSequences in common.h - for (size_t i = 0; i < sampleSize; i++) - sequences[0][i] = static_cast(i); - auto val = Value::Create(sampleShape, sequences, device); + size_t batchCount = 1; + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(1); + input = GenerateSequences(expectedSeqLens, sampleShape); + auto val = Value::Create(sampleShape, input, device); + val->CopyTo(sampleShape, output); - CheckCopyToOutput(sampleSize, sequences, output); + CheckCopyToOutput(sampleSize, input, output); // Check batch of sample. - size_t batchCount = 2; - sequences.clear(); - sequences.resize(2, std::vector(sampleSize)); - for (size_t s = 0; s < batchCount; s++) - { - for (size_t i = 0; i < sampleSize; i++) - { - sequences[s][i] = static_cast(s * 10 + i); - } - } - val = Value::Create(sampleShape, sequences, device); - VerifyException([&val, &sampleShape, &output, &seqLens]() { - val->CopyTo(sampleShape, output, seqLens, false); + batchCount = 2; + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(1); + input = GenerateSequences(expectedSeqLens, sampleShape); + val = Value::Create(sampleShape, input, device); + + // The batch axis is too small + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); }, "The output buffer is too small."); - val->CopyTo(sampleShape, output, seqLens); - CheckCopyToOutput(sampleSize, sequences, output, seqLens); - // Check sequence of sample + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output); + + // Check sequence of sample, but single batch size_t sampleCount = 4; - sequences.clear(); - sequences.resize(1, std::vector(sampleSize * sampleCount)); - for (size_t i = 0; i < sampleSize * sampleCount; i++) - { - sequences[0][i] = static_cast(i); - } - val = Value::Create(sampleShape, sequences, device); - VerifyException([&val, &sampleShape, &output, &seqLens]() { - val->CopyTo(sampleShape, output, seqLens, false); + batchCount = 1; + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCount); + input = GenerateSequences(expectedSeqLens, sampleShape); + val = Value::Create(sampleShape, input, device); + + // The sequence axis is too small + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); }, "The output buffer is too small."); - val->CopyTo(sampleShape, output, seqLens); - CheckCopyToOutput(sampleSize, sequences, output, seqLens); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); // Check batch of sequence of the same length, no mask needed. batchCount = 4; sampleCount = 3; - sequences.clear(); - sequences.resize(batchCount, std::vector(sampleSize * sampleCount)); - for (size_t s = 0; s < batchCount; s++) + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCount); + input = GenerateSequences(expectedSeqLens, sampleShape); + val = Value::Create(sampleShape, input, device); + + // The batch axis is too small, the sequence axis is sufficient. + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); + }, "The output buffer is too small."); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + + // Check batch of sequecnes with different length, mask needed. + std::vector sampleCountList {6, 9, 2}; + batchCount = sampleCountList.size(); + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCountList[i]); + input = GenerateSequences(expectedSeqLens, sampleShape); + val = Value::Create(sampleShape, input, device); + + // The batch axis is sufficient, the sequence axis is too small + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); + }, "The output buffer is too small."); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + + // More batches and sequences + sampleCountList = {6, 12, 2, 1, 5, 3, 4}; + batchCount = sampleCountList.size(); + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCountList[i]); + input = GenerateSequences(expectedSeqLens, sampleShape); + val = Value::Create(sampleShape, input, device); + + // Both the batch and sequence axes are too small. + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); + }, "The output buffer is too small."); + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + + // Random batch and sequence + int testRun = 4; + size_t maxNumOfSequences = 100; + size_t maxSequenceLen = 100; + // This is only used to generate number of sequnces, so boost distribution is not needed. + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, maxNumOfSequences); + for (int i = 0; i < testRun; i++) { - for (size_t i = 0; i < sampleSize * sampleCount; i++) - { - sequences[s][i] = static_cast(s * 10 + i); - } + batchCount = distribution(generator); + + expectedSeqLens = GenerateSequenceLengths(batchCount, maxSequenceLen); + input = GenerateSequences(expectedSeqLens, sampleShape); + val = Value::Create(sampleShape, input, device); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); } - val = Value::Create(sampleShape, sequences, device); - val->CopyTo(sampleShape, output, seqLens); - CheckCopyToOutput(sampleSize, sequences, output, seqLens); +} + +template +void ValueCopyToOneHotTest(const DeviceDescriptor device) +{ + size_t dim = 100; + NDShape sampleShape{{dim}}; + std::vector> input; + std::vector> output; + std::vector expectedSeqLens; + std::vector actualSeqLens; + + // Todo: add tests dense to sparse + // Check single sample. + size_t batchCount = 1; + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(1); + input = GenerateOneHotSequences(expectedSeqLens, dim); + auto val = Value::Create(dim, input, device); + + val->CopyTo(sampleShape, output); + CheckCopyToOutput(1, input, output); + + // Check batch of sample. + batchCount = 2; + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(1); + input = GenerateOneHotSequences(expectedSeqLens, dim); + val = Value::Create(dim, input, device); + + // The batch axis is too small + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); + }, "The output buffer is too small."); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(1, input, output); + + // Check sequence of sample, but single batch + size_t sampleCount = 4; + batchCount = 1; + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCount); + input = GenerateOneHotSequences(expectedSeqLens, dim); + val = Value::Create(dim, input, device); + + // The sequence axis is too small + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); + }, "The output buffer is too small."); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(1, input, output, actualSeqLens); + + // Check batch of sequence of the same length, no mask needed. + batchCount = 4; + sampleCount = 3; + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCount); + input = GenerateOneHotSequences(expectedSeqLens, dim); + val = Value::Create(dim, input, device); + + // The batch axis is too small, the sequence axis is sufficient. + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); + }, "The output buffer is too small."); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(1, input, output, actualSeqLens); // Check batch of sequecnes with different length, mask needed. - std::vector sampleCountList{6, 9, 2, 1, 5, 3, 4}; + std::vector sampleCountList{6, 9, 2}; + batchCount = sampleCountList.size(); + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCountList[i]); + input = GenerateOneHotSequences(expectedSeqLens, dim); + val = Value::Create(dim, input, device); + + // The batch axis is sufficient, the sequence axis is too small + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); + }, "The output buffer is too small."); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(1, input, output, actualSeqLens); + + // More batches and sequences + sampleCountList = {6, 12, 2, 1, 5, 3, 4}; batchCount = sampleCountList.size(); - sequences.clear(); - sequences.resize(batchCount, std::vector(0)); - for (size_t s = 0; s < batchCount; s++) + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCountList[i]); + input = GenerateOneHotSequences(expectedSeqLens, dim); + val = Value::Create(dim, input, device); + + // Both the batch and sequence axes are too small. + VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { + val->CopyTo(sampleShape, output, actualSeqLens, false); + }, "The output buffer is too small."); + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(1, input, output, actualSeqLens); + + // Random batch and sequence + int testRun = 4; + size_t maxNumOfSequences = 100; + size_t maxSequenceLen = 100; + // This is only used to generate number of sequnces, so boost distribution is not needed. + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, maxNumOfSequences); + for (int i = 0; i < testRun; i++) { - for (size_t i = 0; i < sampleSize * sampleCountList[s]; i++) - { - sequences[s].push_back(static_cast(s * 10 + i)); - } + batchCount = distribution(generator); + + expectedSeqLens = GenerateSequenceLengths(batchCount, maxSequenceLen); + input = GenerateOneHotSequences(expectedSeqLens, dim); + val = Value::Create(dim, input, device); + + val->CopyTo(sampleShape, output, actualSeqLens); + CheckCopyToOutput(1, input, output, actualSeqLens); } - val = Value::Create(sampleShape, sequences, device); - val->CopyTo(sampleShape, output, seqLens); - CheckCopyToOutput(sampleSize, sequences, output, seqLens); } void TestSettingParameterValuesManually(const DeviceDescriptor& device) @@ -436,6 +602,7 @@ void SparseSequenceBatchValueCreationTest(size_t vocabSize, size_t maxAllowedSeq void ValueTests() { fprintf(stderr, "\nValueTests..\n"); + srand(1); TestSettingParameterValuesManually(DeviceDescriptor::CPUDevice()); @@ -447,12 +614,12 @@ void ValueTests() ValueCreationOneHotNoNDMaskTest(DeviceDescriptor::CPUDevice(), true); ValueCreationOneHotWithNDMaskTest(DeviceDescriptor::CPUDevice(), false); ValueCreationOneHotWithNDMaskTest(DeviceDescriptor::CPUDevice(), true); -<<<<<<< HEAD SparseSequenceBatchValueCreationTest(300, 7, DeviceDescriptor::CPUDevice()); SparseSequenceBatchValueCreationTest(2300, 1, DeviceDescriptor::CPUDevice()); -======= ValueCopyToDenseTest(DeviceDescriptor::CPUDevice()); ->>>>>>> 0a57cbc... refactor Value::CopyTo to allocate memory in .h file; adapt unit tests; add sequencesLens as parameter for Value::CopyTo + ValueCopyToDenseTest(DeviceDescriptor::CPUDevice()); + ValueCopyToOneHotTest(DeviceDescriptor::CPUDevice()); + ValueCopyToOneHotTest(DeviceDescriptor::CPUDevice()); if (IsGPUAvailable()) { @@ -466,11 +633,11 @@ void ValueTests() ValueCreationOneHotNoNDMaskTest(DeviceDescriptor::GPUDevice(0), true); ValueCreationOneHotWithNDMaskTest(DeviceDescriptor::GPUDevice(0), false); ValueCreationOneHotWithNDMaskTest(DeviceDescriptor::GPUDevice(0), true); -<<<<<<< HEAD SparseSequenceBatchValueCreationTest(50000, 1, DeviceDescriptor::GPUDevice(0)); SparseSequenceBatchValueCreationTest(6000, 6, DeviceDescriptor::GPUDevice(0)); -======= ValueCopyToDenseTest(DeviceDescriptor::GPUDevice(0)); ->>>>>>> 0a57cbc... refactor Value::CopyTo to allocate memory in .h file; adapt unit tests; add sequencesLens as parameter for Value::CopyTo + ValueCopyToDenseTest(DeviceDescriptor::GPUDevice(0)); + ValueCopyToOneHotTest(DeviceDescriptor::GPUDevice(0)); + ValueCopyToOneHotTest(DeviceDescriptor::GPUDevice(0)); } } From 6ff042719558fdc237a2b0c2d26dfb2164b2bb62 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Sun, 18 Dec 2016 19:06:30 +0100 Subject: [PATCH 032/120] add unit tests for CopyTo, improve unit tests to use random sequences, add test for single sample, single sequence in Value.Create fix linux build add missing error message. fix linux build switch order of function definition to make linux happy add instantiation of functions minor changes add comments, using random number in tests restore main.cpp --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 20 +- Source/CNTKv2LibraryDll/Value.cpp | 25 +- Tests/UnitTests/V2LibraryTests/Main.cpp | 10 +- Tests/UnitTests/V2LibraryTests/ValueTests.cpp | 292 ++++++++++-------- 4 files changed, 196 insertions(+), 151 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 3d65d3e195b4..8621cf81738c 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -21,7 +21,6 @@ #include #include #include -#include #ifdef SWIG #define final @@ -947,7 +946,8 @@ namespace CNTK /// /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. /// The sequence buffer is on CPU. - /// The Value should have the same axes as variable. + /// The sequence buffer will be resized if nencessary. + /// The Value should have the same tensor shape as sampleShape. /// template void CopyTo(const NDShape& sampleShape, std::vector>& sequences) @@ -956,17 +956,24 @@ namespace CNTK CopyTo(sampleShape, sequences, seqLens, true); } + /// + /// Same as buffer, except if isResizeable is false, the sequence buffer will not be resized. Instead, an runtime error is thrown. + /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. + /// The sequenceLengths will be resized if necessary, even isResizeable is false. + /// template void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable = true) { CheckAndResizeOutputBuffer(sampleShape.Rank(), sampleShape.TotalSize(), sequences, sequenceLengths, isResizeable); - CopyToVector(sampleShape, sequences, sequenceLengths); + CopyToVector(sampleShape, sequences, sequenceLengths); } /// /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. + /// The output data is of the one hot vector format. /// The sequence buffer is on CPU. - /// The Value should have the same axes as variable. + /// The sequence buffer will be resized if nencessary. + /// The Value should have the same tensor shape as sampleShape. /// void CopyTo(const NDShape& sampleShape, std::vector>& sequences) { @@ -974,6 +981,11 @@ namespace CNTK CopyTo(sampleShape, sequences, seqLens, true); } + /// + /// Same as buffer, except if isResizeable is false, the sequence buffer will not be resized. Instead, an runtime error is thrown. + /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. + /// The sequenceLengths will be resized if necessary, even isResizeable is false. + /// void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable = true) { // For OneHot vector, only 1 value is needed for a sample. diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index 02197cfb30c8..d5c2ef66132f 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -286,7 +286,7 @@ namespace CNTK deviceValueData = valueData; } else - deviceValueData = valueData->DeepClone(device, readOnly); + deviceValueData = valueData->DeepClone(device, readOnly); return MakeSharedObject(deviceValueData, deviceValueMask); } @@ -305,7 +305,7 @@ namespace CNTK InvalidArgument("Value::Create:: The number of elements in the vector containing sequence data must be a multiple of the size of specified sampel shape"); auto sequenceLength = currentSequence.size() / numElementsPerSample; - auto sequenceDataShape = sampleShape.AppendShape({sequenceLength}); + auto sequenceDataShape = sampleShape.AppendShape({ sequenceLength }); sequencesData.push_back(MakeSharedObject(sequenceDataShape, currentSequence)); } @@ -376,10 +376,10 @@ namespace CNTK } template - CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths) + void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths) { if (sampleShape[0] != sampleShape.TotalSize()) - InvalidArgument(""); + InvalidArgument("The sample shape's leading axis dimensionality must equal the total size of the sample for sparse data."); CopyToImpl(sampleShape, sequences, sequenceLengths); } @@ -389,10 +389,11 @@ namespace CNTK std::vector>& sequences, std::vector& sequenceLengths) { - auto valueRank = Shape().Rank(); + auto valueShape = Shape(); + auto valueRank = valueShape.Rank(); auto sampleRank = sampleShape.Rank(); - if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != Shape().SubShape(0, sampleRank))) - RuntimeError("The variable and the Value does not have the same tensor shape."); + if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != valueShape.SubShape(0, sampleRank))) + RuntimeError("The sample shape does not match the value shape."); size_t numOfSequences; size_t maxSequenceLen; @@ -400,16 +401,16 @@ namespace CNTK { // no batch axis, only sequence axis numOfSequences = 1; - maxSequenceLen = Shape()[valueRank - 1]; + maxSequenceLen = valueShape[valueRank - 1]; } else { assert(valueRank == sampleShape.Rank() + 2); - numOfSequences = Shape()[valueRank - 1]; - maxSequenceLen = Shape()[valueRank - 2]; + numOfSequences = valueShape[valueRank - 1]; + maxSequenceLen = valueShape[valueRank - 2]; } - // Check output buffer size + // Check batch size if (sequences.size() < numOfSequences) RuntimeError("The size of output buffer is too small"); @@ -602,5 +603,7 @@ namespace CNTK template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequencesLens); template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequencesLens); + template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); + template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); } diff --git a/Tests/UnitTests/V2LibraryTests/Main.cpp b/Tests/UnitTests/V2LibraryTests/Main.cpp index e1e376f5cc7b..d6ffa5703020 100644 --- a/Tests/UnitTests/V2LibraryTests/Main.cpp +++ b/Tests/UnitTests/V2LibraryTests/Main.cpp @@ -46,13 +46,13 @@ int main() // Lets disable automatic unpacking of PackedValue object to detect any accidental unpacking // which will have a silent performance degradation otherwise - //Internal::SetAutomaticUnpackingOfPackedValues(/*disable =*/ true); + Internal::SetAutomaticUnpackingOfPackedValues(/*disable =*/ true); - //// Note: Run the device selection tests first since later tests - //// may interfere with device selection by freezing default device - //DeviceSelectionTests(); + // Note: Run the device selection tests first since later tests + // may interfere with device selection by freezing default device + DeviceSelectionTests(); - //NDArrayViewTests(); + NDArrayViewTests(); ValueTests(); TensorTests(); FunctionTests(); diff --git a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp index 8066e16a6fe3..f3df8d032720 100644 --- a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp @@ -11,29 +11,11 @@ using namespace CNTK; using namespace std; -// Check the actual shape matches the expected shape, sequence number and sample size. -void CheckShape(const NDShape& shape, const NDShape& expectedShape, const size_t expectedNumOfSequences, const size_t expectedSampleSize) -{ - if (shape != expectedShape) - { - ReportFailure("The shape of the value does not match. Expected: %S, actual: %S\n", expectedShape.AsString().c_str(), shape.AsString().c_str()); - } - size_t numOfSequences = shape[shape.Rank() - 1]; - if (numOfSequences != expectedNumOfSequences) - { - ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedNumOfSequences, numOfSequences); - } - size_t sampleSize = shape.SubShape(0, shape.Rank() - 2).TotalSize(); - if (sampleSize != expectedSampleSize) - { - ReportFailure("The sample size in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedSampleSize, sampleSize); - } -} - // Check the actual Value match the expected shape and the given data (in dense format) template -void CheckValue(const ValuePtr testValue, const NDShape& expectedShape, const size_t expectedSampleSize, const vector>& expectedData, const vector& seqLenList) +void CheckValue(const ValuePtr testValue, const NDShape& sampleShape, const vector>& expectedData, const vector& seqLenList) { + size_t sampleSize = sampleShape.TotalSize(); // Check parameters if (expectedData.size() != seqLenList.size()) { @@ -41,26 +23,46 @@ void CheckValue(const ValuePtr testValue, const NDShape& expectedShape, const si } for (size_t i = 0; i < expectedData.size(); i++) { - if (expectedData[i].size() != seqLenList[i] * expectedSampleSize) + if (expectedData[i].size() != seqLenList[i] * sampleSize) { - ReportFailure("Parameter erroe: the number of data for sequence %" PRIu64 " in the expected data does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", i, seqLenList[i] * expectedSampleSize, expectedData[i].size()); + ReportFailure("Parameter erroe: the number of data for sequence %" PRIu64 " in the expected data does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", i, seqLenList[i] * sampleSize, expectedData[i].size()); } } // Check shape - CheckShape(testValue->Shape(), expectedShape, seqLenList.size(), expectedSampleSize); + auto valueRank = testValue->Shape().Rank(); + auto sampleRank = sampleShape.Rank(); + if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != testValue->Shape().SubShape(0, sampleRank))) + ReportFailure("The Value does not have the expected shape."); - // Get data from Value + size_t numOfSequences; + if (valueRank == sampleShape.Rank() + 1) + { + // no batch axis, only sequence axis + numOfSequences = 1; + } + else + { + assert(valueRank == sampleShape.Rank() + 2); + numOfSequences = testValue->Shape()[valueRank - 1]; + } + + if (numOfSequences != expectedData.size()) + { + ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedData.size(), numOfSequences); + } + + // Get data from Value vector outputData(testValue->Shape().TotalSize()); NDArrayViewPtr arrayOutput = MakeSharedObject(testValue->Shape(), outputData, false); arrayOutput->CopyFrom(*testValue->Data()); size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end()); size_t oIndex = 0; - size_t sampleSize = testValue->Shape().SubShape(0, testValue->Shape().Rank() - 2).TotalSize(); for (size_t seq = 0; seq < seqLenList.size(); seq++) { + // Todo: need to check NDMask is also correct. It is partly done in CopyTo tests. size_t seqLen = seqLenList[seq]; for (size_t sIndex = 0; sIndex < seqLen * sampleSize; sIndex++, oIndex++) { @@ -76,7 +78,7 @@ void CheckValue(const ValuePtr testValue, const NDShape& expectedShape, const si // Check the actual Value match the expected shape and the given data (in onehot vector format) template -void CheckValue(const ValuePtr testValue, const NDShape& expectedShape, const size_t vocabSize, const vector>& expectedData, const vector& seqLenList) +void CheckValue(const ValuePtr testValue, const size_t vocabSize, const vector>& expectedData, const vector& seqLenList) { // Check parameters if (expectedData.size() != seqLenList.size()) @@ -91,12 +93,22 @@ void CheckValue(const ValuePtr testValue, const NDShape& expectedShape, const si } } - // Check shape - CheckShape(testValue->Shape(), expectedShape, seqLenList.size(), vocabSize); + // Check shape + NDShape shape = testValue->Shape(); + size_t valueRank = shape.Rank(); + if (valueRank < 2 || valueRank > 3 || shape[0] != vocabSize) + { + ReportFailure("The shape of the value does not match\n"); + } + size_t numOfSequences = valueRank == 2 ? 1 : shape[2]; + if (numOfSequences != expectedData.size()) + { + ReportFailure("The sequence number in the Value does not match. Expected: %" PRIu64 ", actual: %" PRIu64 ".", expectedData.size(), numOfSequences); + } // Get data from Value - vector outputData(testValue->Shape().TotalSize()); - NDArrayViewPtr arrayOutput = MakeSharedObject(testValue->Shape(), outputData, false); + vector outputData(shape.TotalSize()); + NDArrayViewPtr arrayOutput = MakeSharedObject(shape, outputData, false); arrayOutput->CopyFrom(*testValue->Data()); size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end()); @@ -126,129 +138,140 @@ void CheckValue(const ValuePtr testValue, const NDShape& expectedShape, const si } } -template -void FillDenseMatrixData(vector>& databuf, const vector& seqLenList, const size_t sampleSize) -{ - for (size_t seq = 0; seq < seqLenList.size(); seq++) - { - auto p = new vector(); - databuf.push_back(*p); - size_t seqLen = seqLenList[seq]; - for (size_t sample = 0; sample < seqLen ; sample++) - { - for (size_t element = 0; element < sampleSize; element++) - { - databuf[seq].push_back(static_cast(seq * 1000 + sample * 100 + element)); - } - } - } -} - template void ValueCreationNoNDMaskTest(const DeviceDescriptor device, bool readOnly) { - //Todo: test numberOfSequences == 1: no batch access, need to adapt checkShape(), CheckValue() - size_t numberOfSequences = 5; - size_t seqLen = 4; vector dims{3, 2}; + NDShape sampleShape(dims); + std::vector> data; + ValuePtr testValue; - vector seqLenList; - for (size_t i = 0; i < numberOfSequences; i++) - { - seqLenList.push_back(seqLen); - } - vector> data; - FillDenseMatrixData(data, seqLenList, dims[0] * dims[1]); + // single sequence, single sample + std::vector seqLenList = {1}; + data = GenerateSequences(seqLenList, sampleShape); + testValue = Value::Create(sampleShape, data, device, readOnly); + CheckValue(testValue, sampleShape, data, seqLenList); - // Create the Value object based on the given data and shape. - NDShape sampleShape(dims); - ValuePtr testValue = Value::Create(sampleShape, data, device, readOnly); + // Single sequnce, multiple samples + seqLenList = {2}; + data = GenerateSequences(seqLenList, sampleShape); + testValue = Value::Create(sampleShape, data, device, readOnly); + CheckValue(testValue, sampleShape, data, seqLenList); + + // Batch with sequences - // Check whether the created value matches expected shape and data. - CheckValue(testValue, {dims[0], dims[1], seqLen, numberOfSequences}, dims[0] * dims[1], data, seqLenList); + // Same sequence length for testing no NDMask is needed. + size_t seqLen = 4; + int testRun = 3; + size_t maxNumOfSequences = 60; + // This is only used to generate number of sequnces, so boost distribution is not needed. + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, maxNumOfSequences); + for (int i = 0; i < testRun; i++) + { + size_t numberOfSequences = distribution(generator); + std::vector seqLenList(numberOfSequences, seqLen); + + data = GenerateSequences(seqLenList, sampleShape); + // Create the Value object based on the given data and shape. + testValue = Value::Create(sampleShape, data, device, readOnly); + // Check whether the created value matches expected shape and data. + CheckValue(testValue, sampleShape, data, seqLenList); + } } template void ValueCreationWithNDMaskTest(const DeviceDescriptor device, bool readOnly) { - vector seqLenList{5, 6, 8, 7}; - vector dims{3, 2}; - - size_t numberOfSequences = seqLenList.size(); - vector> data; - FillDenseMatrixData(data, seqLenList, dims[0] * dims[1]); - + vector dims{1, 4}; NDShape sampleShape(dims); - ValuePtr testValue = Value::Create(sampleShape, data, device, readOnly); + size_t numberOfSequences; + size_t maxAllowedSeqLen = 128; + size_t maxSeqLen; + std::vector> data; + std::vector seqLenList; - // Check whether the created value matches expected shape and data. - size_t maxSeqLen = *max_element(seqLenList.begin(), seqLenList.end()); - CheckValue(testValue, {dims[0], dims[1], maxSeqLen, numberOfSequences}, dims[0] * dims[1], data, seqLenList); + size_t maxNumOfSequences = 80; + // This is only used to generate number of sequnces, so boost distribution is not needed. + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, maxNumOfSequences); + int testRun = 3; + for (int i = 0; i < testRun; i++) + { + numberOfSequences = distribution(generator); + seqLenList = GenerateSequenceLengths(numberOfSequences, maxAllowedSeqLen); + maxSeqLen = *std::max_element(seqLenList.begin(), seqLenList.end()); + data = GenerateSequences(seqLenList, sampleShape); + + ValuePtr testValue = Value::Create(sampleShape, data, device, readOnly); + CheckValue(testValue, sampleShape, data, seqLenList); + } } template void ValueCreationOneHotNoNDMaskTest(const DeviceDescriptor device, bool readOnly) { - size_t numberOfSequences = 5; - size_t seqLen = 4; - size_t vocabSize = 16; - - vector seqLenList; - for (size_t i = 0; i < numberOfSequences; i++) - { - seqLenList.push_back(seqLen); - } - vector> data; - for (size_t n = 0; n < numberOfSequences; n++) + size_t vocabSize = 18; + std::vector> data; + ValuePtr testValue; + + // Single sequence, single sample + std::vector seqLenList = {1}; + data = GenerateOneHotSequences(seqLenList, vocabSize); + testValue = Value::Create(vocabSize, data, device, readOnly); + CheckValue(testValue, vocabSize, data, seqLenList); + + // Single sequence, multiple samples + seqLenList = {2}; + data = GenerateOneHotSequences(seqLenList, vocabSize); + testValue = Value::Create(vocabSize, data, device, readOnly); + CheckValue(testValue, vocabSize, data, seqLenList); + + size_t maxNumOfSequences = 160; + size_t seqLen = 26; + // This is only used to generate number of sequnces, so boost distribution is not needed. + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, maxNumOfSequences); + int testRun = 3; + for (int i = 0; i < testRun; i++) { - auto p = new vector(); - data.push_back(*p); - for (size_t s = 0; s < seqLen; s++) - { - data[n].push_back((s * 10 + n) % vocabSize); - } - } - - ValuePtr testValue = Value::Create(vocabSize, data, device, readOnly); + size_t numberOfSequences = distribution(generator); + std::vector seqLenList(numberOfSequences, seqLen); - CheckValue(testValue, {vocabSize, seqLen, numberOfSequences}, vocabSize, data, seqLenList); + data = GenerateOneHotSequences(seqLenList, vocabSize); + testValue = Value::Create(vocabSize, data, device, readOnly); + CheckValue(testValue, vocabSize, data, seqLenList); + } } template void ValueCreationOneHotWithNDMaskTest(const DeviceDescriptor device, bool readOnly) { - vector seqLenList{5, 6, 8, 7}; - size_t maxSeqLen = 0; - size_t vocabSize = 13; - - vector> data; - size_t numberOfSequences = seqLenList.size(); - for (size_t n = 0; n < numberOfSequences; n++) + size_t vocabSize = 64; + size_t numberOfSequences; + size_t maxAllowedSeqLen = 95; + size_t maxSeqLen; + std::vector> data; + std::vector seqLenList; + + size_t maxNumOfSequences = 70; + // This is only used to generate number of sequnces, so boost distribution is not needed. + std::default_random_engine generator; + std::uniform_int_distribution distribution(1, maxNumOfSequences); + int testRun = 3; + for (int i = 0; i < testRun; i++) { - auto p = new vector(); - data.push_back(*p); - size_t seqLen = seqLenList[n]; - maxSeqLen = max(seqLen, maxSeqLen); - for (size_t s = 0; s < seqLen; s++) - { - data[n].push_back((s * 10 + n) % vocabSize); - } + numberOfSequences = distribution(generator); + seqLenList = GenerateSequenceLengths(numberOfSequences, maxAllowedSeqLen); + maxSeqLen = *std::max_element(seqLenList.begin(), seqLenList.end()); + data = GenerateOneHotSequences(seqLenList, vocabSize); + ValuePtr testValue = Value::Create(vocabSize, data, device, readOnly); + CheckValue(testValue, vocabSize, data, seqLenList); } - - ValuePtr testValue = Value::Create(vocabSize, data, device, readOnly); - - CheckValue(testValue, {vocabSize, maxSeqLen, numberOfSequences}, vocabSize, data, seqLenList); } template -void CheckCopyToOutput(const size_t sampleSize, std::vector> expected, std::vector> actual) -{ - std::vector actualSeqLens(0); - CheckCopyToOutput(sampleSize, expected, actual, actualSeqLens); -} - -template -void CheckCopyToOutput(const size_t sampleSize, std::vector>& expected, std::vector>& actual, std::vector& actualSeqLens) +void CheckCopyToOutput(const size_t sampleSize, const std::vector>& expected, const std::vector>& actual, const std::vector& actualSeqLens) { bool useSeqLens; if (actualSeqLens.size() != 0) @@ -278,18 +301,25 @@ void CheckCopyToOutput(const size_t sampleSize, std::vector(i)); } for (size_t j = 0; j < len; j++) { if (expected[i][j] != actual[i][j]) { - ReportFailure("Seq " PRIu64 " does not match.\n", i); + ReportFailure("Seq %lu does not match.\n", static_cast(i)); } } } } +template +void CheckCopyToOutput(const size_t sampleSize, const std::vector>& expected, const std::vector>& actual) +{ + std::vector actualSeqLens(0); + CheckCopyToOutput(sampleSize, expected, actual, actualSeqLens); +} + template void ValueCopyToDenseTest(const DeviceDescriptor device) { @@ -310,7 +340,7 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) auto val = Value::Create(sampleShape, input, device); val->CopyTo(sampleShape, output); - CheckCopyToOutput(sampleSize, input, output); + CheckCopyToOutput(sampleSize, input, output); // Check batch of sample. batchCount = 2; @@ -326,7 +356,7 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) }, "The output buffer is too small."); val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output); + CheckCopyToOutput(sampleSize, input, output); // Check sequence of sample, but single batch size_t sampleCount = 4; @@ -343,7 +373,7 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) }, "The output buffer is too small."); val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); // Check batch of sequence of the same length, no mask needed. batchCount = 4; @@ -360,7 +390,7 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) }, "The output buffer is too small."); val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); // Check batch of sequecnes with different length, mask needed. std::vector sampleCountList {6, 9, 2}; @@ -377,7 +407,7 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) }, "The output buffer is too small."); val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); // More batches and sequences sampleCountList = {6, 12, 2, 1, 5, 3, 4}; @@ -393,7 +423,7 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) val->CopyTo(sampleShape, output, actualSeqLens, false); }, "The output buffer is too small."); val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); // Random batch and sequence int testRun = 4; @@ -411,7 +441,7 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) val = Value::Create(sampleShape, input, device); val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); } } From 88ecb8d4ffe3b655108f302af69fc3838cfac416 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Mon, 19 Dec 2016 11:07:51 +0100 Subject: [PATCH 033/120] update CopyTo for OneHot using vocabSize instead of shape --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 14 ++++---- Source/CNTKv2LibraryDll/Value.cpp | 11 +++--- Tests/UnitTests/V2LibraryTests/ValueTests.cpp | 35 +++++++++---------- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 8621cf81738c..aa5076a3b293 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -975,10 +975,10 @@ namespace CNTK /// The sequence buffer will be resized if nencessary. /// The Value should have the same tensor shape as sampleShape. /// - void CopyTo(const NDShape& sampleShape, std::vector>& sequences) + void CopyTo(const size_t vocabularySize, std::vector>& sequences) { std::vector seqLens; - CopyTo(sampleShape, sequences, seqLens, true); + CopyTo(vocabularySize, sequences, seqLens, true); } /// @@ -986,18 +986,18 @@ namespace CNTK /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. /// The sequenceLengths will be resized if necessary, even isResizeable is false. /// - void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable = true) + void CopyTo(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable = true) { // For OneHot vector, only 1 value is needed for a sample. - CheckAndResizeOutputBuffer(sampleShape.Rank(), 1, sequences, sequenceLengths, isResizeable); + CheckAndResizeOutputBuffer(1, 1, sequences, sequenceLengths, isResizeable); auto dataType = GetDataType(); if (dataType == DataType::Float) { - CopyToVector(sampleShape, sequences, sequenceLengths); + CopyToVector(vocabularySize, sequences, sequenceLengths); } else if (dataType == DataType::Double) { - CopyToVector(sampleShape, sequences, sequenceLengths); + CopyToVector(vocabularySize, sequences, sequenceLengths); } } @@ -1011,7 +1011,7 @@ namespace CNTK CNTK_API void CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); template - CNTK_API void CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); + CNTK_API void CopyToVector(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths); template void CopyToImpl(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index d5c2ef66132f..f6026eb014e4 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -376,12 +376,9 @@ namespace CNTK } template - void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths) + void Value::CopyToVector(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths) { - if (sampleShape[0] != sampleShape.TotalSize()) - InvalidArgument("The sample shape's leading axis dimensionality must equal the total size of the sample for sparse data."); - - CopyToImpl(sampleShape, sequences, sequenceLengths); + CopyToImpl({{vocabularySize}}, sequences, sequenceLengths); } template @@ -603,7 +600,7 @@ namespace CNTK template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequencesLens); template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequencesLens); - template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); - template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); + template CNTK_API void Value::CopyToVector(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths); + template CNTK_API void Value::CopyToVector(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths); } diff --git a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp index f3df8d032720..91c32ba5fc31 100644 --- a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp @@ -449,7 +449,6 @@ template void ValueCopyToOneHotTest(const DeviceDescriptor device) { size_t dim = 100; - NDShape sampleShape{{dim}}; std::vector> input; std::vector> output; std::vector expectedSeqLens; @@ -464,7 +463,7 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) input = GenerateOneHotSequences(expectedSeqLens, dim); auto val = Value::Create(dim, input, device); - val->CopyTo(sampleShape, output); + val->CopyTo(dim, output); CheckCopyToOutput(1, input, output); // Check batch of sample. @@ -476,11 +475,11 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) val = Value::Create(dim, input, device); // The batch axis is too small - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); + VerifyException([&val, &dim, &output, &actualSeqLens]() { + val->CopyTo(dim, output, actualSeqLens, false); }, "The output buffer is too small."); - val->CopyTo(sampleShape, output, actualSeqLens); + val->CopyTo(dim, output, actualSeqLens); CheckCopyToOutput(1, input, output); // Check sequence of sample, but single batch @@ -493,11 +492,11 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) val = Value::Create(dim, input, device); // The sequence axis is too small - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); + VerifyException([&val, &dim, &output, &actualSeqLens]() { + val->CopyTo(dim, output, actualSeqLens, false); }, "The output buffer is too small."); - val->CopyTo(sampleShape, output, actualSeqLens); + val->CopyTo(dim, output, actualSeqLens); CheckCopyToOutput(1, input, output, actualSeqLens); // Check batch of sequence of the same length, no mask needed. @@ -510,11 +509,11 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) val = Value::Create(dim, input, device); // The batch axis is too small, the sequence axis is sufficient. - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); + VerifyException([&val, &dim, &output, &actualSeqLens]() { + val->CopyTo(dim, output, actualSeqLens, false); }, "The output buffer is too small."); - val->CopyTo(sampleShape, output, actualSeqLens); + val->CopyTo(dim, output, actualSeqLens); CheckCopyToOutput(1, input, output, actualSeqLens); // Check batch of sequecnes with different length, mask needed. @@ -527,11 +526,11 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) val = Value::Create(dim, input, device); // The batch axis is sufficient, the sequence axis is too small - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); + VerifyException([&val, &dim, &output, &actualSeqLens]() { + val->CopyTo(dim, output, actualSeqLens, false); }, "The output buffer is too small."); - val->CopyTo(sampleShape, output, actualSeqLens); + val->CopyTo(dim, output, actualSeqLens); CheckCopyToOutput(1, input, output, actualSeqLens); // More batches and sequences @@ -544,10 +543,10 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) val = Value::Create(dim, input, device); // Both the batch and sequence axes are too small. - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); + VerifyException([&val, &dim, &output, &actualSeqLens]() { + val->CopyTo(dim, output, actualSeqLens, false); }, "The output buffer is too small."); - val->CopyTo(sampleShape, output, actualSeqLens); + val->CopyTo(dim, output, actualSeqLens); CheckCopyToOutput(1, input, output, actualSeqLens); // Random batch and sequence @@ -565,7 +564,7 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) input = GenerateOneHotSequences(expectedSeqLens, dim); val = Value::Create(dim, input, device); - val->CopyTo(sampleShape, output, actualSeqLens); + val->CopyTo(dim, output, actualSeqLens); CheckCopyToOutput(1, input, output, actualSeqLens); } } From 69c75c4494afdb6e28c3599634f84c811a5e450b Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 20 Dec 2016 07:35:17 +0100 Subject: [PATCH 034/120] Using Variable as parameter to CopyTo to infer dynamic axes correctly; Add check for PackedValue; Simplify processing based on assumption that sequence starts at 0 and ends at the first invalid mask; other changes from CR. add tests for variable with 0,1 or 2 dynamic axes, and adapt tests to new API, refactor tests to share code restore main and tests make class Value after class Variable to make linux happy fix bug in test --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 620 +++++++++--------- Source/CNTKv2LibraryDll/Value.cpp | 201 +++--- Tests/UnitTests/V2LibraryTests/ValueTests.cpp | 237 ++++--- 3 files changed, 536 insertions(+), 522 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index aa5076a3b293..c6c7af6adc9e 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -789,307 +789,6 @@ namespace CNTK std::shared_ptr> m_matrixView; }; - /// - /// Denotes a multi-dimensional array with an optional mask and is the actual data fed into or produced from a computation. - /// The mask is typically lower dimensionality than the data, meaning data is masked in coarse individual sample units where - /// sample shape is data.Shape().SubShape(0, data.Shape().Rank() - mask.Shape().Rank) - /// Also, note that the size of the data's trailing mask.Shape().Rank() dimensions must match the mask shape dimensions. - /// - class Value : public std::enable_shared_from_this - { - public: - /// - /// A multi-dimensional value with no mask. - /// - CNTK_API Value(const NDArrayViewPtr& data); - - /// - /// A multi-dimensional value with an associated mask. - /// - CNTK_API Value(const NDArrayViewPtr& data, const NDMaskPtr& mask); - - /// - /// Create a new Value object containing a collection of variable length sequences. - /// The sequenceStartFlags argument allows specifying for each sequence whether that sequence is a - /// a new sequence or continuation of a previous sequence at the same index in the - /// sequences vector from a previous call to this method. - /// The created Value object contains a copy of the specified 'sequences' data. - /// - template - CNTK_API static ValuePtr Create(const NDShape& sampleShape, const std::vector>& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly = false); - - /// - /// Create a new Value object containing a collection of variable length sequences. - /// The created Value object contains a copy of the specified 'sequences' data. - /// - template - static ValuePtr Create(const NDShape& sampleShape, const std::vector>& sequences, const DeviceDescriptor& device, bool readOnly = false) - { - return Create(sampleShape, sequences, {}, device, readOnly); - } - - /// - /// Create a new Value object containing a collection of variable length sequences. - /// The created Value object contains a copy of the specified 'sequences' data. - /// - static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly = false) - { - return Create(sampleShape, sequences, sequenceStartFlags, device, readOnly, /*createNewCopy =*/ false); - } - - /// - /// Create a new Value object containing a collection of variable length sequences. - /// The created Value object contains a copy of the specified 'sequences' data. - /// - static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const DeviceDescriptor& device, bool readOnly = false) - { - return Create(sampleShape, sequences, {}, device, readOnly); - } - - /// - /// Create a new Value object containing a collection of variable length sequences of one hot vectors - /// The created Value object contains a copy of the specified 'sequences' data. - /// - template - CNTK_API static ValuePtr Create(size_t vocabularySize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly = false); - - /// - /// Create a new Value object containing a collection of variable length sequences of one hot vectors - /// The created Value object contains a copy of the specified 'sequences' data. - /// - template - static ValuePtr Create(size_t vocabularySize, const std::vector>& oneHotSequences, const DeviceDescriptor& device, bool readOnly = false) - { - return Create(vocabularySize, oneHotSequences, {}, device, readOnly); - } - - /// - /// Destruct 'this' Value object. - /// - virtual ~Value(); - - /// - /// Returns the descriptor of the device that 'this' Value resides on - /// - virtual DeviceDescriptor Device() const { return m_data->Device(); } - - /// - /// Returns the data type of 'this' Value's contents. - /// - virtual DataType GetDataType() const { return m_data->GetDataType(); } - - /// - /// Returns the storage format of 'this' Value. - /// - virtual StorageFormat GetStorageFormat() const { return m_data->GetStorageFormat(); } - - /// - /// Returns the shape 'this' Value. - /// - virtual const NDShape& Shape() const { return m_data->Shape(); } - - /// - /// Returns a boolean indicating if 'this' Value contains data in sparse storage format. - /// - bool IsSparse() const - { - return (GetStorageFormat() != StorageFormat::Dense); - } - - /// - /// Returns a boolean indicating if 'this' Value is read-only. - /// - virtual bool IsReadOnly() const { return m_data->IsReadOnly(); } - - /// - /// Returns the number of masked/invalid values - /// - virtual size_t MaskedCount() const - { - return m_mask ? m_mask->MaskedCount() : 0; - } - - /// - /// Returns the NDArrayView object corresponding to the data contents of 'this value object. - /// - virtual NDArrayViewPtr Data() const; - - /// - /// Returns the NDMask object corresponding to the mask associated with 'this value object. - /// - virtual NDMaskPtr Mask() const; - - /// - /// Creates a new Value with newly allocated storage on the same device as 'this' Value and copies 'this' Value's contents into the newly allocated Value. - /// - virtual ValuePtr DeepClone(bool readOnly) const; - - /// - /// Creates a new Value with newly allocated storage on the same device as 'this' Value and copies 'this' Value's contents into the newly allocated Value. - /// - ValuePtr DeepClone() const - { - return DeepClone(IsReadOnly()); - } - - /// - /// Creates a new Value which is an alias of 'this' Value. - /// - virtual ValuePtr Alias(bool readOnly = false) const; - - /// - /// Copies the contents of the 'source' Value to 'this' Value. - /// The shapes of the 'source' Value's data and mask must be identical to 'this' Value's data and mask. - /// - virtual void CopyFrom(const Value& source); - - /// - /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. - /// The sequence buffer is on CPU. - /// The sequence buffer will be resized if nencessary. - /// The Value should have the same tensor shape as sampleShape. - /// - template - void CopyTo(const NDShape& sampleShape, std::vector>& sequences) - { - std::vector seqLens; - CopyTo(sampleShape, sequences, seqLens, true); - } - - /// - /// Same as buffer, except if isResizeable is false, the sequence buffer will not be resized. Instead, an runtime error is thrown. - /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. - /// The sequenceLengths will be resized if necessary, even isResizeable is false. - /// - template - void CopyTo(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable = true) - { - CheckAndResizeOutputBuffer(sampleShape.Rank(), sampleShape.TotalSize(), sequences, sequenceLengths, isResizeable); - CopyToVector(sampleShape, sequences, sequenceLengths); - } - - /// - /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. - /// The output data is of the one hot vector format. - /// The sequence buffer is on CPU. - /// The sequence buffer will be resized if nencessary. - /// The Value should have the same tensor shape as sampleShape. - /// - void CopyTo(const size_t vocabularySize, std::vector>& sequences) - { - std::vector seqLens; - CopyTo(vocabularySize, sequences, seqLens, true); - } - - /// - /// Same as buffer, except if isResizeable is false, the sequence buffer will not be resized. Instead, an runtime error is thrown. - /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. - /// The sequenceLengths will be resized if necessary, even isResizeable is false. - /// - void CopyTo(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable = true) - { - // For OneHot vector, only 1 value is needed for a sample. - CheckAndResizeOutputBuffer(1, 1, sequences, sequenceLengths, isResizeable); - auto dataType = GetDataType(); - if (dataType == DataType::Float) - { - CopyToVector(vocabularySize, sequences, sequenceLengths); - } - else if (dataType == DataType::Double) - { - CopyToVector(vocabularySize, sequences, sequenceLengths); - } - } - - private: - template - static void AppendSparseSequenceData(const NDArrayViewPtr& sequenceData, std::vector& colStarts, std::vector& rowIndices, std::vector& nonZeroValues, size_t maxSequenceLength); - - CNTK_API static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly, bool createNewCopy); - - template - CNTK_API void CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); - - template - CNTK_API void CopyToVector(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths); - - template - void CopyToImpl(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths); - - template - void CheckAndResizeOutputBuffer(const size_t sampleRank, const size_t sampleSize, std::vector>& sequences, std::vector& sequenceLengths, bool isResizeable) - { - auto valueRank = Shape().Rank(); - size_t numOfSequences; - size_t maxSequenceLen; - - if (valueRank == sampleRank + 1) - { - // no batch axis, only sequence axis - numOfSequences = 1; - maxSequenceLen = Shape()[valueRank - 1]; - } - else - { - assert(valueRank == sampleRank + 2); - numOfSequences = Shape()[valueRank - 1]; - maxSequenceLen = Shape()[valueRank - 2]; - } - - // resize the sequnce length buffer to reflect the number of sequences in output. - if (sequenceLengths.size() < numOfSequences) - sequenceLengths.resize(numOfSequences); - - // Check whether the additional space in the sequences output buffer needs to be allocated if it is resizeable. - if (isResizeable) - { - const MaskKind* maskData = nullptr; - NDMaskPtr cpuMask = nullptr; - if (Mask() != nullptr) - { - cpuMask = (Device() != DeviceDescriptor::CPUDevice()) ? Mask()->DeepClone(DeviceDescriptor::CPUDevice()) : Mask(); - maskData = cpuMask->DataBuffer(); - } - - size_t sampleCount, seqStart; - for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) - { - if (maskData == nullptr) - { - sampleCount = maxSequenceLen; - } - else - { - seqStart = seqIndex * maxSequenceLen; - sampleCount = 0; - for (size_t i = 0; i < maxSequenceLen; i++) - if (maskData[seqStart + i] != MaskKind::Invalid) - sampleCount++; - } - - if (seqIndex < sequences.size()) - { - if (sequences[seqIndex].size() < sampleCount * sampleSize) - sequences[seqIndex].resize(sampleCount * sampleSize); - } - else - { - assert(seqIndex == sequences.size()); - auto p = new std::vector(sampleCount * sampleSize); - sequences.push_back(*p); - } - } - } - } - - // Disallow copy and move construction and assignment - Value(const Value&) = delete; Value& operator=(const Value&) = delete; Value(Value&&) = delete; Value& operator=(Value&&) = delete; - - protected: - mutable NDArrayViewPtr m_data; - mutable NDMaskPtr m_mask; - }; - /// /// Denotes an Axis of a Variable and is used for specifying the axes parameters of certain Functions such as reductions. /// Besides the static axes corresponding to each of the axes of the Variable's shape, Variables of kind 'Input' and any @@ -2297,6 +1996,325 @@ namespace std { namespace CNTK { + /// + /// Denotes a multi-dimensional array with an optional mask and is the actual data fed into or produced from a computation. + /// The mask is typically lower dimensionality than the data, meaning data is masked in coarse individual sample units where + /// sample shape is data.Shape().SubShape(0, data.Shape().Rank() - mask.Shape().Rank) + /// Also, note that the size of the data's trailing mask.Shape().Rank() dimensions must match the mask shape dimensions. + /// + class Value : public std::enable_shared_from_this + { + public: + /// + /// A multi-dimensional value with no mask. + /// + CNTK_API Value(const NDArrayViewPtr& data); + + /// + /// A multi-dimensional value with an associated mask. + /// + CNTK_API Value(const NDArrayViewPtr& data, const NDMaskPtr& mask); + + /// + /// Create a new Value object containing a collection of variable length sequences. + /// The sequenceStartFlags argument allows specifying for each sequence whether that sequence is a + /// a new sequence or continuation of a previous sequence at the same index in the + /// sequences vector from a previous call to this method. + /// The created Value object contains a copy of the specified 'sequences' data. + /// + template + CNTK_API static ValuePtr Create(const NDShape& sampleShape, const std::vector>& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly = false); + + /// + /// Create a new Value object containing a collection of variable length sequences. + /// The created Value object contains a copy of the specified 'sequences' data. + /// + template + static ValuePtr Create(const NDShape& sampleShape, const std::vector>& sequences, const DeviceDescriptor& device, bool readOnly = false) + { + return Create(sampleShape, sequences, {}, device, readOnly); + } + + /// + /// Create a new Value object containing a collection of variable length sequences. + /// The created Value object contains a copy of the specified 'sequences' data. + /// + static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly = false) + { + return Create(sampleShape, sequences, sequenceStartFlags, device, readOnly, /*createNewCopy =*/ false); + } + + /// + /// Create a new Value object containing a collection of variable length sequences. + /// The created Value object contains a copy of the specified 'sequences' data. + /// + static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const DeviceDescriptor& device, bool readOnly = false) + { + return Create(sampleShape, sequences, {}, device, readOnly); + } + + /// + /// Create a new Value object containing a collection of variable length sequences of one hot vectors + /// The created Value object contains a copy of the specified 'sequences' data. + /// + template + CNTK_API static ValuePtr Create(size_t vocabularySize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly = false); + + /// + /// Create a new Value object containing a collection of variable length sequences of one hot vectors + /// The created Value object contains a copy of the specified 'sequences' data. + /// + template + static ValuePtr Create(size_t vocabularySize, const std::vector>& oneHotSequences, const DeviceDescriptor& device, bool readOnly = false) + { + return Create(vocabularySize, oneHotSequences, {}, device, readOnly); + } + + /// + /// Destruct 'this' Value object. + /// + virtual ~Value(); + + /// + /// Returns the descriptor of the device that 'this' Value resides on + /// + virtual DeviceDescriptor Device() const { + return m_data->Device(); + } + + /// + /// Returns the data type of 'this' Value's contents. + /// + virtual DataType GetDataType() const { + return m_data->GetDataType(); + } + + /// + /// Returns the storage format of 'this' Value. + /// + virtual StorageFormat GetStorageFormat() const { + return m_data->GetStorageFormat(); + } + + /// + /// Returns the shape 'this' Value. + /// + virtual const NDShape& Shape() const { + return m_data->Shape(); + } + + /// + /// Returns a boolean indicating if 'this' Value contains data in sparse storage format. + /// + bool IsSparse() const + { + return (GetStorageFormat() != StorageFormat::Dense); + } + + /// + /// Returns a boolean indicating if 'this' Value is read-only. + /// + virtual bool IsReadOnly() const { + return m_data->IsReadOnly(); + } + + /// + /// Returns the number of masked/invalid values + /// + virtual size_t MaskedCount() const + { + return m_mask ? m_mask->MaskedCount() : 0; + } + + /// + /// Returns the NDArrayView object corresponding to the data contents of 'this value object. + /// + virtual NDArrayViewPtr Data() const; + + /// + /// Returns the NDMask object corresponding to the mask associated with 'this value object. + /// + virtual NDMaskPtr Mask() const; + + /// + /// Creates a new Value with newly allocated storage on the same device as 'this' Value and copies 'this' Value's contents into the newly allocated Value. + /// + virtual ValuePtr DeepClone(bool readOnly) const; + + /// + /// Creates a new Value with newly allocated storage on the same device as 'this' Value and copies 'this' Value's contents into the newly allocated Value. + /// + ValuePtr DeepClone() const + { + return DeepClone(IsReadOnly()); + } + + /// + /// Creates a new Value which is an alias of 'this' Value. + /// + virtual ValuePtr Alias(bool readOnly = false) const; + + /// + /// Copies the contents of the 'source' Value to 'this' Value. + /// The shapes of the 'source' Value's data and mask must be identical to 'this' Value's data and mask. + /// + virtual void CopyFrom(const Value& source); + + /// + /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. + /// The sequence buffer will be resized if necessary. + /// The Value should have the same tensor shape as sampleShape. + /// + template + void CopyTo(const Variable& sampleVariable, std::vector>& sequences) + { + std::vector seqLens; + CopyTo(sampleVariable, sequences, seqLens, true); + } + + /// + /// Same as above, except if isResizable is false, the sequence buffer will not be resized. Instead, a runtime error is thrown. + /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. + /// The sequenceLengths will be resized if necessary, even isResizable is false. + /// + template + void CopyTo(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable = true) + { + ResizeOutputBuffer(sampleVariable, sampleVariable.Shape().TotalSize(), sequences, sequenceLengths, isResizable); + CopyToVector(sampleVariable, sequences, sequenceLengths); + } + + /// + /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. + /// The output data is of the one hot vector format. + /// The sequence buffer is on CPU. + /// The sequence buffer will be resized if nencessary. + /// The Value should have the same tensor shape as sampleShape. + /// + void CopyTo(const Variable& sampleVariable, std::vector>& sequences) + { + std::vector seqLens; + CopyTo(sampleVariable, sequences, seqLens, true); + } + + /// + /// Same as buffer, except if isResizable is false, the sequence buffer will not be resized. Instead, an runtime error is thrown. + /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. + /// The sequenceLengths will be resized if necessary, even isResizable is false. + /// + void CopyTo(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable = true) + { + // For OneHot vector, only 1 value is needed for a sample. + ResizeOutputBuffer(sampleVariable, 1, sequences, sequenceLengths, isResizable); + auto dataType = GetDataType(); + if (dataType == DataType::Float) + { + CopyToVector(sampleVariable, sequences, sequenceLengths); + } + else if (dataType == DataType::Double) + { + CopyToVector(sampleVariable, sequences, sequenceLengths); + } + } + + private: + template + static void AppendSparseSequenceData(const NDArrayViewPtr& sequenceData, std::vector& colStarts, std::vector& rowIndices, std::vector& nonZeroValues, size_t maxSequenceLength); + + CNTK_API static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly, bool createNewCopy); + + template + CNTK_API void CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); + + template + CNTK_API void CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); + + template + void CopyToImpl(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); + + virtual std::pair GetSequenceAndBatchLength(const Variable& sampleVariable); + + template + void ResizeOutputBuffer(const Variable& sampleVariable, size_t sampleSize, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable) + { + size_t numOfSequences; + size_t maxSequenceLen; + std::tie(maxSequenceLen, numOfSequences) = GetSequenceAndBatchLength(sampleVariable); + + // resize the sequnce length buffer to reflect the number of sequences in output. + if (sequenceLengths.size() < numOfSequences) + sequenceLengths.resize(numOfSequences); + + // Check whether the output buffer is sufficient and allocate new space if isResizable is true. + const MaskKind* maskData = nullptr; + NDMaskPtr cpuMask = nullptr; + if (Mask() != nullptr) + { + cpuMask = (Device().Type() != DeviceKind::CPU) ? Mask()->DeepClone(DeviceDescriptor::CPUDevice()) : Mask(); + maskData = cpuMask->DataBuffer(); + } + + size_t sampleCount = 0, seqStart; + if (sequences.size() < numOfSequences) + { + if (isResizable) + sequences.resize(numOfSequences); + else + RuntimeError("The size of output buffer is too small"); + } + + for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) + { + if (maskData == nullptr) + { + sampleCount = maxSequenceLen; + } + else + { + seqStart = seqIndex * maxSequenceLen; + if (maskData[seqStart] == MaskKind::Invalid) + RuntimeError("No leading invalid mask is allowed for a sequence."); + // The assumption here is that a sequence always start at 0 (no invaid mark at the beginning), + // and ends at the first invalid mask. + bool isEnd = false; + sampleCount = 0; + for (size_t i = 0; i < maxSequenceLen; i++) + { + if (!isEnd) + { + if (maskData[seqStart + i] != MaskKind::Invalid) + sampleCount++; + else + isEnd = true; + } + if (isEnd && (maskData[seqStart + i] != MaskKind::Invalid)) + RuntimeError("Invalid sequence."); + } + assert(isEnd || (maskData[seqStart + maxSequenceLen - 1] != MaskKind::Invalid && sampleCount == maxSequenceLen)); + } + sequenceLengths[seqIndex] = sampleCount; + + if (sequences[seqIndex].size() < sampleCount * sampleSize) + { + if (isResizable) + sequences[seqIndex].resize(sampleCount * sampleSize); + else + RuntimeError("The size of output buffer is too small"); + } + } + + for (size_t i = numOfSequences; i < sequenceLengths.size(); i++) + sequenceLengths[i] = 0; + } + + // Disallow copy and move construction and assignment + Value(const Value&) = delete; Value& operator=(const Value&) = delete; Value(Value&&) = delete; Value& operator=(Value&&) = delete; + + protected: + mutable NDArrayViewPtr m_data; + mutable NDMaskPtr m_mask; + }; + /// /// Encapsulates the internal computation state of a Function computed as part of the 'Forward' call on a Function /// that must be passed to a subsequent 'Backward' call on the same Function to backpropagate gradient values diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index f6026eb014e4..5d623c422476 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -360,79 +360,51 @@ namespace CNTK } template - void DirectCopy(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest, size_t& destSampleStart); + void DirectCopy(const ElementType *source, size_t sampleCount, size_t sampleSize, std::vector& dest); template - void CopyDenseToOneHot(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest, size_t& destSampleStart); + void CopyDenseToOneHot(const ElementType *source, size_t sampleCount, size_t sampleSize, std::vector& dest); template - void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequenceLengths) + void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths) { // Check the data type matches if (AsDataType() != GetDataType()) InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); - CopyToImpl(sampleShape, sequences, sequenceLengths); + CopyToImpl(sampleVariable, sequences, sequenceLengths); } template - void Value::CopyToVector(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths) + void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths) { - CopyToImpl({{vocabularySize}}, sequences, sequenceLengths); + CopyToImpl(sampleVariable, sequences, sequenceLengths); } template - void Value::CopyToImpl(const NDShape& sampleShape, + void Value::CopyToImpl(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths) { - auto valueShape = Shape(); - auto valueRank = valueShape.Rank(); - auto sampleRank = sampleShape.Rank(); - if ((valueRank < sampleRank + 1) || (valueRank > sampleRank + 2) || (sampleShape != valueShape.SubShape(0, sampleRank))) - RuntimeError("The sample shape does not match the value shape."); + // Todo: support packed value by unpacking? + if (dynamic_cast(this) != nullptr) + RuntimeError("CopyTo() does not support PackedValue yet."); size_t numOfSequences; size_t maxSequenceLen; - if (valueRank == sampleShape.Rank() + 1) - { - // no batch axis, only sequence axis - numOfSequences = 1; - maxSequenceLen = valueShape[valueRank - 1]; - } - else - { - assert(valueRank == sampleShape.Rank() + 2); - numOfSequences = valueShape[valueRank - 1]; - maxSequenceLen = valueShape[valueRank - 2]; - } + std::tie(maxSequenceLen, numOfSequences) = GetSequenceAndBatchLength(sampleVariable); - // Check batch size if (sequences.size() < numOfSequences) RuntimeError("The size of output buffer is too small"); - // Check sequenceLengths size. - if (sequenceLengths.size() < numOfSequences) - { - RuntimeError("The size of sequenceLengths does not match."); - } - else - { - for (size_t i = numOfSequences; i < sequenceLengths.size(); i++) - sequenceLengths[i] = 0; - } - // Copy data to the CPU device if required. const ValueType *valueData; - const MaskKind* maskData; NDArrayViewPtr cpuArrayView; - NDMaskPtr cpuMask; - if (Device() != DeviceDescriptor::CPUDevice()) + if (Device().Type() != DeviceKind::CPU) { // Todo: leverage sparse if the original NDArrayView is in spase. cpuArrayView = MakeSharedObject(GetDataType(), Data()->Shape(), DeviceDescriptor::CPUDevice()); cpuArrayView->CopyFrom(*Data()); - cpuMask = Mask() != nullptr ? Mask()->DeepClone(DeviceDescriptor::CPUDevice()) : nullptr; } else { @@ -446,63 +418,68 @@ namespace CNTK { cpuArrayView = Data(); } - cpuMask = Mask(); } valueData = cpuArrayView->DataBuffer(); - maskData = cpuMask != nullptr ? cpuMask->DataBuffer() : nullptr; - auto sampleSize = sampleShape.TotalSize(); + auto sampleSize = sampleVariable.Shape().TotalSize(); for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) { size_t seqStart = seqIndex * maxSequenceLen; - size_t destSampleCount = 0; - if (maskData == nullptr) + + // The assumption here is that a sequence always start at 0 (no invaid mark at the beginning), + // and ends at the first invalid mask. + // Therefore, no need to check NDMask again. The SequenceLengths already contains the correct number of samples. + + // Todo: if function pointer or lambda could support template, switch to use them. + if (typeid(DestType) == typeid(size_t)) { - // Todo: if function pointer or lambda could support template, switch to use them. - if (typeid(DestType) == typeid(size_t)) - { - CopyDenseToOneHot(valueData + seqStart * sampleSize, maxSequenceLen, sampleSize, sequences[seqIndex], destSampleCount); - } - else - { - DirectCopy(valueData + seqStart * sampleSize, maxSequenceLen, sampleSize, sequences[seqIndex], destSampleCount); - } - sequenceLengths[seqIndex] = destSampleCount; + CopyDenseToOneHot(valueData + seqStart * sampleSize, sequenceLengths[seqIndex], sampleSize, sequences[seqIndex]); } else { - // NDMask is not null - size_t current = seqStart; - size_t seqEnd = seqStart + maxSequenceLen; - while (current < seqEnd) - { - // find first valid mask. - while ((current < seqEnd) && (maskData[current] == MaskKind::Invalid)) - current++; - auto sampleStart = current; - - // find the next invalid mask. - while ((current < seqEnd) && (maskData[current] != MaskKind::Invalid)) - current++; - assert(current >= sampleStart); - if (current > sampleStart) - { - // Todo: if function pointer or lambda could support template, switch to use them. - if (typeid(DestType) == typeid(size_t)) - { - CopyDenseToOneHot(valueData + seqStart * sampleSize, current - sampleStart, sampleSize, sequences[seqIndex], destSampleCount); - } - else - { - DirectCopy(valueData + seqStart * sampleSize, current - sampleStart, sampleSize, sequences[seqIndex], destSampleCount); - } - } - } - sequenceLengths[seqIndex] = destSampleCount; + DirectCopy(valueData + seqStart * sampleSize, sequenceLengths[seqIndex], sampleSize, sequences[seqIndex]); } } } + std::pair Value::GetSequenceAndBatchLength(const Variable& sampleVariable) + { + size_t sampleRank = sampleVariable.Shape().Rank(); + size_t maxSequenceLength = 1; + size_t numSequences = 1; + + if (Shape().Rank() < sampleRank) + RuntimeError("The value shape has less ranks than the variable shape."); + size_t maskRank = Shape().Rank() - sampleRank; + + if (sampleVariable.Shape() != Shape().SubShape(0, sampleRank)) + RuntimeError("The shape of the sampleVariable does not match the value shape."); + + if (maskRank > 2) + RuntimeError("Only 2 dynamic axes are supported now."); + else if (maskRank == 2) + { + // Only 2 axes are supported at the moment, sequence axis should be the first and batch axis the second. + maxSequenceLength = Shape()[sampleRank]; + numSequences = Shape()[sampleRank + 1]; + } + else if (maskRank == 1) + { + if (sampleVariable.DynamicAxes().size() > 1) + { + maxSequenceLength = Shape()[sampleRank]; + } + else + { + // there's only one axis (the default batch axis). + numSequences = Shape()[sampleRank]; + } + } + + return std::pair(maxSequenceLength, numSequences); + } + + void PackedValue::Unpack() const { if (m_packedDataLayout && (m_packedDataLayout->GetNumTimeSteps() != 1) && (m_packedDataLayout->GetNumSequences() != 1) && Internal::IsAutomaticUnpackingOfPackedValuesDisabled()) @@ -537,20 +514,19 @@ namespace CNTK } template - void DirectCopy(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest, size_t& destSampleStart) + void DirectCopy(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest) { if (typeid(ElementType) != typeid(DestType)) RuntimeError("Source and destination must be the same data type."); DestType *destData = dest.data(); - if ((destSampleStart + sampleCount) * sampleSize > dest.size()) + if (sampleCount * sampleSize > dest.size()) RuntimeError("The output buffer is too small."); - std::copy(source, source + sampleCount * sampleSize, reinterpret_cast(destData + destSampleStart * sampleSize)); - destSampleStart += sampleCount; + std::copy(source, source + sampleCount * sampleSize, reinterpret_cast(destData)); } template - void CopyDenseToOneHot(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest, size_t& destSampleStart) + void CopyDenseToOneHot(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest) { if (typeid(DestType) != typeid(size_t)) { @@ -559,36 +535,28 @@ namespace CNTK const ElementType *currentp = source; const ElementType *lastp = source + sampleCount * sampleSize; + size_t destIndex = 0; while (currentp < lastp) { - auto sampleEndp = currentp + sampleSize; - auto indexp = std::find_if(currentp, sampleEndp, [](const ElementType val) { - return val != 0; - }); - - if (indexp == sampleEndp) - { - RuntimeError("Cannot convert to onehot vector: the sample does not have any non-zero value."); - } - else + size_t index = sampleSize; + bool found = false; + for (size_t i = 0; i < sampleSize; i++) { - if (std::find_if(indexp + 1, sampleEndp, [](const ElementType val) { - return val != 0; - }) != sampleEndp) + if (*currentp == 1) { - RuntimeError("Cannot convert to onehot vector: more than one non-zero value in the sample."); - } - else - { - if (destSampleStart >= dest.size()) - RuntimeError("The output buffer is too small."); - else - { - dest[destSampleStart++] = static_cast(indexp - currentp); - } + if (found) + RuntimeError("Cannot convert to onehot vector: more than one non-zero value in the sample."); + index = i; + found = true; } + else if (*currentp != 0) + RuntimeError("Cannot convert to onehot vector: contain value other than 0 and 1."); + currentp++; } - currentp += sampleSize; + if (!found) + RuntimeError("Cannot convert to onehot vector: the sample does not have any non-zero value."); + assert(index != sampleSize); + dest[destIndex++] = static_cast(index); } assert(currentp == lastp); } @@ -598,9 +566,8 @@ namespace CNTK template /*static*/ CNTK_API ValuePtr Value::Create(const NDShape& sampleShape, const std::vector>& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); - template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequencesLens); - template CNTK_API void Value::CopyToVector(const NDShape& sampleShape, std::vector>& sequences, std::vector& sequencesLens); - template CNTK_API void Value::CopyToVector(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths); - template CNTK_API void Value::CopyToVector(const size_t vocabularySize, std::vector>& sequences, std::vector& sequenceLengths); - + template CNTK_API void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequencesLens); + template CNTK_API void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequencesLens); + template CNTK_API void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); + template CNTK_API void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); } diff --git a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp index 91c32ba5fc31..2713a2b778b8 100644 --- a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp @@ -321,7 +321,63 @@ void CheckCopyToOutput(const size_t sampleSize, const std::vector -void ValueCopyToDenseTest(const DeviceDescriptor device) +Variable CreateVariable(NDShape shape, int numOfDynamicAxes) +{ + std::vector dynamicAxes; + + switch (numOfDynamicAxes) + { + case 0: + dynamicAxes = {}; + break; + case 1: + dynamicAxes = {Axis::DefaultBatchAxis()}; // If only 1 dynamic axis, it is treated as batch axis + break; + case 2: + dynamicAxes = {Axis::DefaultDynamicAxis(), Axis::DefaultBatchAxis()}; // The first is sequence, and the second is batch. + break; + default: + RuntimeError("No more than 2 dynamic axes is allowed."); + } + + Variable sampleVariable(shape, VariableKind::Output, AsDataType(), nullptr, false, + dynamicAxes, false /*bool isSparse*/, L"sampleVariable", L"sampleVariableUid"); + + return sampleVariable; +} + +template +void TestResizeDense(const Variable& sampleVariable, std::vector& expectedSeqLens, std::vector>& output, std::vector& actualSeqLens, const DeviceDescriptor& device) +{ + auto input = GenerateSequences(expectedSeqLens, sampleVariable.Shape()); + auto val = Value::Create(sampleVariable.Shape(), input, device); + + // The sequence axis is too small + VerifyException([&val, &sampleVariable, &output, &actualSeqLens]() { + val->CopyTo(sampleVariable, output, actualSeqLens, false); + }, "The exception 'output buffer is too small.' not thrown."); + + val->CopyTo(sampleVariable, output, actualSeqLens); + CheckCopyToOutput(sampleVariable.Shape().TotalSize(), input, output, actualSeqLens); +} + +template +void TestResizeOneHot(const Variable& sampleVariable, std::vector& expectedSeqLens, std::vector>& output, std::vector& actualSeqLens, const DeviceDescriptor& device) +{ + auto input = GenerateOneHotSequences(expectedSeqLens, sampleVariable.Shape().TotalSize()); + auto val = Value::Create(sampleVariable.Shape().TotalSize(), input, device); + + // The sequence axis is too small + VerifyException([&val, &sampleVariable, &output, &actualSeqLens]() { + val->CopyTo(sampleVariable, output, actualSeqLens, false); + }, "The exception 'output buffer is too small.' not thrown."); + + val->CopyTo(sampleVariable, output, actualSeqLens); + CheckCopyToOutput(1, input, output, actualSeqLens); +} + +template +void ValueCopyToDenseTest(const DeviceDescriptor& device) { NDShape sampleShape{{2, 3}}; auto sampleSize = sampleShape.TotalSize(); @@ -331,7 +387,8 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) std::vector actualSeqLens; //Todo: add tests sparse to dense. - // Check single sample. + // Check single sample. No dynamic axis for the sampleVariable + auto sampleVariable = CreateVariable(sampleShape, 0); size_t batchCount = 1; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) @@ -339,41 +396,52 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) input = GenerateSequences(expectedSeqLens, sampleShape); auto val = Value::Create(sampleShape, input, device); - val->CopyTo(sampleShape, output); - CheckCopyToOutput(sampleSize, input, output); + val->CopyTo(sampleVariable, output); + CheckCopyToOutput(sampleSize, input, output); + + // 1 dynamic axis (as batch) for the sampleVariable + sampleVariable = CreateVariable(sampleShape, 1); + val->CopyTo(sampleVariable, output); + CheckCopyToOutput(sampleSize, input, output); + + // 2 dynamic axes for the sampleVariable + sampleVariable = CreateVariable(sampleShape, 2); + val->CopyTo(sampleVariable, output); + CheckCopyToOutput(sampleSize, input, output); // Check batch of sample. + // 1 dynamic axis (as batch) for the sampleVariable + sampleVariable = CreateVariable(sampleShape, 1); batchCount = 2; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(1); - input = GenerateSequences(expectedSeqLens, sampleShape); - val = Value::Create(sampleShape, input, device); + input = GenerateSequences(expectedSeqLens, sampleVariable.Shape()); + val = Value::Create(sampleVariable.Shape(), input, device); + + // The sequence axis is too small + // Not using TestResizeDense as the input value is needed for the next test. + VerifyException([&val, &sampleVariable, &output, &actualSeqLens]() { + val->CopyTo(sampleVariable, output, actualSeqLens, false); + }, "The exception 'output buffer is too small.' not thrown."); - // The batch axis is too small - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); - }, "The output buffer is too small."); + val->CopyTo(sampleVariable, output, actualSeqLens); + CheckCopyToOutput(sampleVariable.Shape().TotalSize(), input, output, actualSeqLens); - val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output); + // 2 dynamic axes for the sampleVariable + sampleVariable = CreateVariable(sampleShape, 2); + val->CopyTo(sampleVariable, output); + CheckCopyToOutput(sampleSize, input, output); // Check sequence of sample, but single batch + // The variable should have 2 dynamic axes. + sampleVariable = CreateVariable(sampleShape, 2); size_t sampleCount = 4; batchCount = 1; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCount); - input = GenerateSequences(expectedSeqLens, sampleShape); - val = Value::Create(sampleShape, input, device); - - // The sequence axis is too small - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); - }, "The output buffer is too small."); - - val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + TestResizeDense(sampleVariable, expectedSeqLens, output, actualSeqLens, device); // Check batch of sequence of the same length, no mask needed. batchCount = 4; @@ -381,16 +449,7 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCount); - input = GenerateSequences(expectedSeqLens, sampleShape); - val = Value::Create(sampleShape, input, device); - - // The batch axis is too small, the sequence axis is sufficient. - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); - }, "The output buffer is too small."); - - val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + TestResizeDense(sampleVariable, expectedSeqLens, output, actualSeqLens, device); // Check batch of sequecnes with different length, mask needed. std::vector sampleCountList {6, 9, 2}; @@ -398,32 +457,15 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCountList[i]); - input = GenerateSequences(expectedSeqLens, sampleShape); - val = Value::Create(sampleShape, input, device); + TestResizeDense(sampleVariable, expectedSeqLens, output, actualSeqLens, device); - // The batch axis is sufficient, the sequence axis is too small - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); - }, "The output buffer is too small."); - - val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); - - // More batches and sequences + // More batches and sequences, need resize sampleCountList = {6, 12, 2, 1, 5, 3, 4}; batchCount = sampleCountList.size(); expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCountList[i]); - input = GenerateSequences(expectedSeqLens, sampleShape); - val = Value::Create(sampleShape, input, device); - - // Both the batch and sequence axes are too small. - VerifyException([&val, &sampleShape, &output, &actualSeqLens]() { - val->CopyTo(sampleShape, output, actualSeqLens, false); - }, "The output buffer is too small."); - val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + TestResizeDense(sampleVariable, expectedSeqLens, output, actualSeqLens, device); // Random batch and sequence int testRun = 4; @@ -440,15 +482,16 @@ void ValueCopyToDenseTest(const DeviceDescriptor device) input = GenerateSequences(expectedSeqLens, sampleShape); val = Value::Create(sampleShape, input, device); - val->CopyTo(sampleShape, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + val->CopyTo(sampleVariable, output, actualSeqLens); + CheckCopyToOutput(sampleSize, input, output, actualSeqLens); } } template -void ValueCopyToOneHotTest(const DeviceDescriptor device) +void ValueCopyToOneHotTest(const DeviceDescriptor& device) { size_t dim = 100; + NDShape sampleShape{{dim}}; std::vector> input; std::vector> output; std::vector expectedSeqLens; @@ -456,6 +499,7 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) // Todo: add tests dense to sparse // Check single sample. + auto sampleVariable = CreateVariable(sampleShape, 0); size_t batchCount = 1; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) @@ -463,41 +507,52 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) input = GenerateOneHotSequences(expectedSeqLens, dim); auto val = Value::Create(dim, input, device); - val->CopyTo(dim, output); + val->CopyTo(sampleVariable, output); + CheckCopyToOutput(1, input, output); + + // 1 dynamic axis (as batch) for the sampleVariable + sampleVariable = CreateVariable(sampleShape, 1); + val->CopyTo(sampleVariable, output); + CheckCopyToOutput(1, input, output); + + // 2 dynamic axes for the sampleVariable + sampleVariable = CreateVariable(sampleShape, 2); + val->CopyTo(sampleVariable, output); CheckCopyToOutput(1, input, output); // Check batch of sample. + // 1 dynamic axis (as batch) for the sampleVariable + sampleVariable = CreateVariable(sampleShape, 1); batchCount = 2; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(1); - input = GenerateOneHotSequences(expectedSeqLens, dim); - val = Value::Create(dim, input, device); + input = GenerateOneHotSequences(expectedSeqLens, sampleVariable.Shape().TotalSize()); + val = Value::Create(sampleVariable.Shape().TotalSize(), input, device); + + // The sequence axis is too small + // Not using TestResize() as the input is needed for the next test. + VerifyException([&val, &sampleVariable, &output, &actualSeqLens]() { + val->CopyTo(sampleVariable, output, actualSeqLens, false); + }, "The exception 'output buffer is too small.' not thrown."); - // The batch axis is too small - VerifyException([&val, &dim, &output, &actualSeqLens]() { - val->CopyTo(dim, output, actualSeqLens, false); - }, "The output buffer is too small."); + val->CopyTo(sampleVariable, output, actualSeqLens); + CheckCopyToOutput(1, input, output, actualSeqLens); - val->CopyTo(dim, output, actualSeqLens); + // 2 dynamic axes for the sampleVariable + sampleVariable = CreateVariable(sampleShape, 2); + val->CopyTo(sampleVariable, output); CheckCopyToOutput(1, input, output); // Check sequence of sample, but single batch + // The variable should have 2 dynamic axes. + sampleVariable = CreateVariable(sampleShape, 2); size_t sampleCount = 4; batchCount = 1; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCount); - input = GenerateOneHotSequences(expectedSeqLens, dim); - val = Value::Create(dim, input, device); - - // The sequence axis is too small - VerifyException([&val, &dim, &output, &actualSeqLens]() { - val->CopyTo(dim, output, actualSeqLens, false); - }, "The output buffer is too small."); - - val->CopyTo(dim, output, actualSeqLens); - CheckCopyToOutput(1, input, output, actualSeqLens); + TestResizeOneHot(sampleVariable, expectedSeqLens, output, actualSeqLens, device); // Check batch of sequence of the same length, no mask needed. batchCount = 4; @@ -505,16 +560,7 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCount); - input = GenerateOneHotSequences(expectedSeqLens, dim); - val = Value::Create(dim, input, device); - - // The batch axis is too small, the sequence axis is sufficient. - VerifyException([&val, &dim, &output, &actualSeqLens]() { - val->CopyTo(dim, output, actualSeqLens, false); - }, "The output buffer is too small."); - - val->CopyTo(dim, output, actualSeqLens); - CheckCopyToOutput(1, input, output, actualSeqLens); + TestResizeOneHot(sampleVariable, expectedSeqLens, output, actualSeqLens, device); // Check batch of sequecnes with different length, mask needed. std::vector sampleCountList{6, 9, 2}; @@ -522,32 +568,15 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCountList[i]); - input = GenerateOneHotSequences(expectedSeqLens, dim); - val = Value::Create(dim, input, device); - - // The batch axis is sufficient, the sequence axis is too small - VerifyException([&val, &dim, &output, &actualSeqLens]() { - val->CopyTo(dim, output, actualSeqLens, false); - }, "The output buffer is too small."); - - val->CopyTo(dim, output, actualSeqLens); - CheckCopyToOutput(1, input, output, actualSeqLens); + TestResizeOneHot(sampleVariable, expectedSeqLens, output, actualSeqLens, device); - // More batches and sequences + // More batches and sequences, resize required sampleCountList = {6, 12, 2, 1, 5, 3, 4}; batchCount = sampleCountList.size(); expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCountList[i]); - input = GenerateOneHotSequences(expectedSeqLens, dim); - val = Value::Create(dim, input, device); - - // Both the batch and sequence axes are too small. - VerifyException([&val, &dim, &output, &actualSeqLens]() { - val->CopyTo(dim, output, actualSeqLens, false); - }, "The output buffer is too small."); - val->CopyTo(dim, output, actualSeqLens); - CheckCopyToOutput(1, input, output, actualSeqLens); + TestResizeOneHot(sampleVariable, expectedSeqLens, output, actualSeqLens, device); // Random batch and sequence int testRun = 4; @@ -564,7 +593,7 @@ void ValueCopyToOneHotTest(const DeviceDescriptor device) input = GenerateOneHotSequences(expectedSeqLens, dim); val = Value::Create(dim, input, device); - val->CopyTo(dim, output, actualSeqLens); + val->CopyTo(sampleVariable, output, actualSeqLens); CheckCopyToOutput(1, input, output, actualSeqLens); } } From 646b8b57562592caeba8f8d624edcfe28975f218 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 20 Dec 2016 17:25:34 +0100 Subject: [PATCH 035/120] rename sampleVariable to outputVariable; add check for shape[0] in onehot. --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 36 ++++++++++----------- Source/CNTKv2LibraryDll/Value.cpp | 38 +++++++++++++---------- 2 files changed, 39 insertions(+), 35 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index c6c7af6adc9e..e788bbf47f2e 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -2163,13 +2163,13 @@ namespace CNTK /// /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. /// The sequence buffer will be resized if necessary. - /// The Value should have the same tensor shape as sampleShape. + /// The Value should have the same tensor shape as outputVariable. /// template - void CopyTo(const Variable& sampleVariable, std::vector>& sequences) + void CopyTo(const Variable& outputVariable, std::vector>& sequences) { std::vector seqLens; - CopyTo(sampleVariable, sequences, seqLens, true); + CopyTo(outputVariable, sequences, seqLens, true); } /// @@ -2178,10 +2178,10 @@ namespace CNTK /// The sequenceLengths will be resized if necessary, even isResizable is false. /// template - void CopyTo(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable = true) + void CopyTo(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable = true) { - ResizeOutputBuffer(sampleVariable, sampleVariable.Shape().TotalSize(), sequences, sequenceLengths, isResizable); - CopyToVector(sampleVariable, sequences, sequenceLengths); + ResizeOutputBuffer(outputVariable, outputVariable.Shape().TotalSize(), sequences, sequenceLengths, isResizable); + CopyToVector(outputVariable, sequences, sequenceLengths); } /// @@ -2191,10 +2191,10 @@ namespace CNTK /// The sequence buffer will be resized if nencessary. /// The Value should have the same tensor shape as sampleShape. /// - void CopyTo(const Variable& sampleVariable, std::vector>& sequences) + void CopyTo(const Variable& outputVariable, std::vector>& sequences) { std::vector seqLens; - CopyTo(sampleVariable, sequences, seqLens, true); + CopyTo(outputVariable, sequences, seqLens, true); } /// @@ -2202,18 +2202,18 @@ namespace CNTK /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. /// The sequenceLengths will be resized if necessary, even isResizable is false. /// - void CopyTo(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable = true) + void CopyTo(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable = true) { // For OneHot vector, only 1 value is needed for a sample. - ResizeOutputBuffer(sampleVariable, 1, sequences, sequenceLengths, isResizable); + ResizeOutputBuffer(outputVariable, 1, sequences, sequenceLengths, isResizable); auto dataType = GetDataType(); if (dataType == DataType::Float) { - CopyToVector(sampleVariable, sequences, sequenceLengths); + CopyToVector(outputVariable, sequences, sequenceLengths); } else if (dataType == DataType::Double) { - CopyToVector(sampleVariable, sequences, sequenceLengths); + CopyToVector(outputVariable, sequences, sequenceLengths); } } @@ -2224,22 +2224,22 @@ namespace CNTK CNTK_API static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly, bool createNewCopy); template - CNTK_API void CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); + CNTK_API void CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); template - CNTK_API void CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); + CNTK_API void CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); template - void CopyToImpl(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); + void CopyToImpl(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); - virtual std::pair GetSequenceAndBatchLength(const Variable& sampleVariable); + virtual std::pair GetSequenceAndBatchLength(const Variable& outputVariable); template - void ResizeOutputBuffer(const Variable& sampleVariable, size_t sampleSize, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable) + void ResizeOutputBuffer(const Variable& outputVariable, size_t sampleSize, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable) { size_t numOfSequences; size_t maxSequenceLen; - std::tie(maxSequenceLen, numOfSequences) = GetSequenceAndBatchLength(sampleVariable); + std::tie(maxSequenceLen, numOfSequences) = GetSequenceAndBatchLength(outputVariable); // resize the sequnce length buffer to reflect the number of sequences in output. if (sequenceLengths.size() < numOfSequences) diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index 5d623c422476..e3d002b80e08 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -366,23 +366,27 @@ namespace CNTK void CopyDenseToOneHot(const ElementType *source, size_t sampleCount, size_t sampleSize, std::vector& dest); template - void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths) + void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths) { // Check the data type matches if (AsDataType() != GetDataType()) InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); - CopyToImpl(sampleVariable, sequences, sequenceLengths); + CopyToImpl(outputVariable, sequences, sequenceLengths); } template - void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths) + void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths) { - CopyToImpl(sampleVariable, sequences, sequenceLengths); + if (outputVariable.Shape()[0] != outputVariable.Shape().TotalSize()) + { + InvalidArgument("The sample variable's leading axis dimensionality must equal the total size of the variable for sparse data."); + } + CopyToImpl(outputVariable, sequences, sequenceLengths); } template - void Value::CopyToImpl(const Variable& sampleVariable, + void Value::CopyToImpl(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths) { @@ -392,7 +396,7 @@ namespace CNTK size_t numOfSequences; size_t maxSequenceLen; - std::tie(maxSequenceLen, numOfSequences) = GetSequenceAndBatchLength(sampleVariable); + std::tie(maxSequenceLen, numOfSequences) = GetSequenceAndBatchLength(outputVariable); if (sequences.size() < numOfSequences) RuntimeError("The size of output buffer is too small"); @@ -421,7 +425,7 @@ namespace CNTK } valueData = cpuArrayView->DataBuffer(); - auto sampleSize = sampleVariable.Shape().TotalSize(); + auto sampleSize = outputVariable.Shape().TotalSize(); for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) { size_t seqStart = seqIndex * maxSequenceLen; @@ -442,18 +446,18 @@ namespace CNTK } } - std::pair Value::GetSequenceAndBatchLength(const Variable& sampleVariable) + std::pair Value::GetSequenceAndBatchLength(const Variable& outputVariable) { - size_t sampleRank = sampleVariable.Shape().Rank(); + size_t sampleRank = outputVariable.Shape().Rank(); size_t maxSequenceLength = 1; size_t numSequences = 1; if (Shape().Rank() < sampleRank) - RuntimeError("The value shape has less ranks than the variable shape."); + RuntimeError("The Value'rank should be greater than or equal to the variable's rank."); size_t maskRank = Shape().Rank() - sampleRank; - if (sampleVariable.Shape() != Shape().SubShape(0, sampleRank)) - RuntimeError("The shape of the sampleVariable does not match the value shape."); + if (outputVariable.Shape() != Shape().SubShape(0, sampleRank)) + RuntimeError("The shape of the outputVariable does not match the value shape."); if (maskRank > 2) RuntimeError("Only 2 dynamic axes are supported now."); @@ -465,7 +469,7 @@ namespace CNTK } else if (maskRank == 1) { - if (sampleVariable.DynamicAxes().size() > 1) + if (outputVariable.DynamicAxes().size() > 1) { maxSequenceLength = Shape()[sampleRank]; } @@ -566,8 +570,8 @@ namespace CNTK template /*static*/ CNTK_API ValuePtr Value::Create(const NDShape& sampleShape, const std::vector>& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); - template CNTK_API void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequencesLens); - template CNTK_API void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequencesLens); - template CNTK_API void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); - template CNTK_API void Value::CopyToVector(const Variable& sampleVariable, std::vector>& sequences, std::vector& sequenceLengths); + template CNTK_API void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequencesLens); + template CNTK_API void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequencesLens); + template CNTK_API void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); + template CNTK_API void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); } From 8cb238dba821f3c81e6712eab56b8f0fe6af6142 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Wed, 21 Dec 2016 10:03:25 +0100 Subject: [PATCH 036/120] make GetNumTimeStepsAndSequences a stand-alone function; use is_same instead of typeid; minor changes --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 14 +++--- Source/CNTKv2LibraryDll/Utils.cpp | 60 ++++++++++++----------- Source/CNTKv2LibraryDll/Utils.h | 2 + Source/CNTKv2LibraryDll/Value.cpp | 32 +++--------- 4 files changed, 48 insertions(+), 60 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index e788bbf47f2e..22202d7d8957 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -2174,7 +2174,7 @@ namespace CNTK /// /// Same as above, except if isResizable is false, the sequence buffer will not be resized. Instead, a runtime error is thrown. - /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. + /// In addition, on return, sequenceLengths contains the length of each sequence in sequences. /// The sequenceLengths will be resized if necessary, even isResizable is false. /// template @@ -2186,10 +2186,9 @@ namespace CNTK /// /// Copy the data stored in the Value object to the buffer 'sequences' as a collection of variable length sequences. - /// The output data is of the one hot vector format. - /// The sequence buffer is on CPU. - /// The sequence buffer will be resized if nencessary. - /// The Value should have the same tensor shape as sampleShape. + /// The output data is in one-hot format. + /// The sequence buffer will be resized if ncessary. + /// The Value should have the same tensor shape as outputVariable. /// void CopyTo(const Variable& outputVariable, std::vector>& sequences) { @@ -2198,7 +2197,7 @@ namespace CNTK } /// - /// Same as buffer, except if isResizable is false, the sequence buffer will not be resized. Instead, an runtime error is thrown. + /// Same as buffer, except if isResizable is false, the sequence buffer will not be resized. Instead, a runtime error is thrown. /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. /// The sequenceLengths will be resized if necessary, even isResizable is false. /// @@ -2274,7 +2273,7 @@ namespace CNTK seqStart = seqIndex * maxSequenceLen; if (maskData[seqStart] == MaskKind::Invalid) RuntimeError("No leading invalid mask is allowed for a sequence."); - // The assumption here is that a sequence always start at 0 (no invaid mark at the beginning), + // The assumption here is that a sequence always start at 0 (no invalid mark at the beginning), // and ends at the first invalid mask. bool isEnd = false; sampleCount = 0; @@ -2287,6 +2286,7 @@ namespace CNTK else isEnd = true; } + // Todo: remove the check below if the assumption above is confirmed. if (isEnd && (maskData[seqStart + i] != MaskKind::Invalid)) RuntimeError("Invalid sequence."); } diff --git a/Source/CNTKv2LibraryDll/Utils.cpp b/Source/CNTKv2LibraryDll/Utils.cpp index 097bd6197933..ffceb129f0d7 100644 --- a/Source/CNTKv2LibraryDll/Utils.cpp +++ b/Source/CNTKv2LibraryDll/Utils.cpp @@ -456,6 +456,34 @@ namespace CNTK #endif } + std::pair GetNumTimeStepsAndSequences(const NDShape& maskShape, size_t numDynamicAxes) + { + size_t maxNumTimeSteps = 1; + size_t numSequences = 1; + if (maskShape.Rank() > 1) + { + // since only 2 axes are supported at the moment, sequence axis should be the first and batch axis -- the second. + // sequence axis dimension determines the maximum number of time steps (= maximum sequence length), + // batch axis dimension -- the number of sequences (= 'training units') in a batch. + maxNumTimeSteps = maskShape[0]; + numSequences = maskShape[1]; + } + else if (maskShape.Rank() > 0) + { + if (numDynamicAxes > 1) + { + maxNumTimeSteps = maskShape[0]; + } + else + { + // there's only one axis (the default batch axis). + numSequences = maskShape[0]; + } + } + + return std::pair(maxNumTimeSteps, numSequences); + } + template std::pair>, MBLayoutPtr> Utils::GetCNTKImplMatrixAndMBLayoutFromValueObject(const Variable& var, const ValuePtr& value) { @@ -505,45 +533,19 @@ namespace CNTK auto mask = value->Mask(); if ((mask != nullptr) && ((varShape.Rank() + mask->Shape().Rank()) != valueShape.Rank())) InvalidArgument("Invalid Value object; the sum of the rank of the mask and data does not equal the Variable's rank + number of dynamic axes"); - - auto getNumTimeStepsAndSequencesFunc = [numDynamicAxes](const NDShape& maskShape, size_t numDynamicAxes) { - size_t maxNumTimeSteps = 1; - size_t numSequences = 1; - if (maskShape.Rank() > 1) - { - // since only 2 axes are supported at the moment, sequence axis should be the first and batch axis -- the second. - // sequence axis dimension determines the maximum number of time steps (= maximum sequence length), - // batch axis dimension -- the number of sequences (= 'training units') in a batch. - maxNumTimeSteps = maskShape[0]; - numSequences = maskShape[1]; - } - else if (maskShape.Rank() > 0) - { - if (numDynamicAxes > 1) - { - maxNumTimeSteps = maskShape[0]; - } - else - { - // there's only one axis (the default batch axis). - numSequences = maskShape[0]; - } - } - return std::pair(maxNumTimeSteps, numSequences); - }; size_t maxNumTimeSteps, numSequences; - std::tie(maxNumTimeSteps, numSequences) = getNumTimeStepsAndSequencesFunc(valueShape.SubShape(varShape.Rank()), numDynamicAxes); + std::tie(maxNumTimeSteps, numSequences) = GetNumTimeStepsAndSequences(valueShape.SubShape(varShape.Rank()), numDynamicAxes); - auto getSequenceStartsAndLengthsFunc = [&getNumTimeStepsAndSequencesFunc](const NDMaskPtr& mask, std::vector& sequenceBeginIndices, std::vector& sequenceLengths, size_t numDynamicAxes) { + auto getSequenceStartsAndLengthsFunc = [](const NDMaskPtr& mask, std::vector& sequenceBeginIndices, std::vector& sequenceLengths, size_t numDynamicAxes) { auto cpuMask = mask; if (mask->Device() != DeviceDescriptor::CPUDevice()) cpuMask = mask->DeepClone(DeviceDescriptor::CPUDevice()); const MaskKind* maskBuffer = cpuMask->DataBuffer(); size_t maxNumTimeSteps, numSequences; - std::tie(maxNumTimeSteps, numSequences) = getNumTimeStepsAndSequencesFunc(mask->Shape(), numDynamicAxes); + std::tie(maxNumTimeSteps, numSequences) = GetNumTimeStepsAndSequences(mask->Shape(), numDynamicAxes); for (size_t i = 0; i < numSequences; ++i) { diff --git a/Source/CNTKv2LibraryDll/Utils.h b/Source/CNTKv2LibraryDll/Utils.h index ac8d9534e8b3..391c68b58325 100644 --- a/Source/CNTKv2LibraryDll/Utils.h +++ b/Source/CNTKv2LibraryDll/Utils.h @@ -493,6 +493,8 @@ namespace CNTK std::string ToString(const std::wstring& wstring); std::wstring ToWString(const std::string& string); + std::pair GetNumTimeStepsAndSequences(const NDShape& maskShape, size_t numDynamicAxes); + // Helper class to manage a collection of learners. class Learners { diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index e3d002b80e08..5f221b157b55 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -407,7 +407,7 @@ namespace CNTK if (Device().Type() != DeviceKind::CPU) { // Todo: leverage sparse if the original NDArrayView is in spase. - cpuArrayView = MakeSharedObject(GetDataType(), Data()->Shape(), DeviceDescriptor::CPUDevice()); + cpuArrayView = MakeSharedObject(GetDataType(), Shape(), DeviceDescriptor::CPUDevice()); cpuArrayView->CopyFrom(*Data()); } else @@ -415,7 +415,7 @@ namespace CNTK // Todo: direct process sparse data without copy if (GetStorageFormat() != StorageFormat::Dense) { - cpuArrayView = MakeSharedObject(GetDataType(), Data()->Shape(), DeviceDescriptor::CPUDevice()); + cpuArrayView = MakeSharedObject(GetDataType(), Shape(), DeviceDescriptor::CPUDevice()); cpuArrayView->CopyFrom(*Data()); } else @@ -435,7 +435,7 @@ namespace CNTK // Therefore, no need to check NDMask again. The SequenceLengths already contains the correct number of samples. // Todo: if function pointer or lambda could support template, switch to use them. - if (typeid(DestType) == typeid(size_t)) + if (std::is_same::value) { CopyDenseToOneHot(valueData + seqStart * sampleSize, sequenceLengths[seqIndex], sampleSize, sequences[seqIndex]); } @@ -454,31 +454,15 @@ namespace CNTK if (Shape().Rank() < sampleRank) RuntimeError("The Value'rank should be greater than or equal to the variable's rank."); - size_t maskRank = Shape().Rank() - sampleRank; + size_t maskRank = Shape().Rank() - sampleRank; if (outputVariable.Shape() != Shape().SubShape(0, sampleRank)) RuntimeError("The shape of the outputVariable does not match the value shape."); if (maskRank > 2) RuntimeError("Only 2 dynamic axes are supported now."); - else if (maskRank == 2) - { - // Only 2 axes are supported at the moment, sequence axis should be the first and batch axis the second. - maxSequenceLength = Shape()[sampleRank]; - numSequences = Shape()[sampleRank + 1]; - } - else if (maskRank == 1) - { - if (outputVariable.DynamicAxes().size() > 1) - { - maxSequenceLength = Shape()[sampleRank]; - } - else - { - // there's only one axis (the default batch axis). - numSequences = Shape()[sampleRank]; - } - } + + std::tie(maxSequenceLength, numSequences) = GetNumTimeStepsAndSequences(Shape().SubShape(sampleRank), outputVariable.DynamicAxes().size()); return std::pair(maxSequenceLength, numSequences); } @@ -520,7 +504,7 @@ namespace CNTK template void DirectCopy(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest) { - if (typeid(ElementType) != typeid(DestType)) + if (!std::is_same::value) RuntimeError("Source and destination must be the same data type."); DestType *destData = dest.data(); @@ -532,7 +516,7 @@ namespace CNTK template void CopyDenseToOneHot(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest) { - if (typeid(DestType) != typeid(size_t)) + if (!std::is_same::value) { RuntimeError("The destination data type must be size_t."); } From 321c6a1efb8f55b8299a033575d86b8b7d4f8e56 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Wed, 21 Dec 2016 13:19:42 +0100 Subject: [PATCH 037/120] remove check for PackedValue as the PackedValue will be automatically unpacked when accessing Data() and Mask(). --- Source/CNTKv2LibraryDll/Value.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index 5f221b157b55..0f0d84c5bcf5 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -390,10 +390,7 @@ namespace CNTK std::vector>& sequences, std::vector& sequenceLengths) { - // Todo: support packed value by unpacking? - if (dynamic_cast(this) != nullptr) - RuntimeError("CopyTo() does not support PackedValue yet."); - + // PackedValue should be automatically unpacked when accessing Data() and Mask(). size_t numOfSequences; size_t maxSequenceLen; std::tie(maxSequenceLen, numOfSequences) = GetSequenceAndBatchLength(outputVariable); From 750dccbfd3b041d41aaf4f738f2e58ed2eb32fe7 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Thu, 5 Jan 2017 13:19:02 +0100 Subject: [PATCH 038/120] remove isResizable and SequenceLength parameters from CopyTO until we have clear understanding for the use case; rename CopyTo to CopyVairiableValueTo; adapt test cases fix error fix errors in tests; add tests for exceptions add more check in GetSequenceAndBatchLength(); Address CR comments --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 178 +++++------- Source/CNTKv2LibraryDll/Value.cpp | 64 +++-- Tests/UnitTests/V2LibraryTests/ValueTests.cpp | 271 ++++++++++-------- 3 files changed, 258 insertions(+), 255 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 22202d7d8957..e2971b8ce287 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -1637,7 +1637,7 @@ namespace CNTK private: #ifdef SWIGCSHARP public: - // Todo: a better way to get hash value? + // TODO: a better way to get hash value? size_t GetHashValue() { return std::hash()(m_dataFields.get()); @@ -2078,53 +2078,37 @@ namespace CNTK /// /// Returns the descriptor of the device that 'this' Value resides on /// - virtual DeviceDescriptor Device() const { - return m_data->Device(); - } + virtual DeviceDescriptor Device() const { return m_data->Device(); } /// /// Returns the data type of 'this' Value's contents. /// - virtual DataType GetDataType() const { - return m_data->GetDataType(); - } + virtual DataType GetDataType() const { return m_data->GetDataType(); } /// /// Returns the storage format of 'this' Value. /// - virtual StorageFormat GetStorageFormat() const { - return m_data->GetStorageFormat(); - } + virtual StorageFormat GetStorageFormat() const { return m_data->GetStorageFormat(); } /// /// Returns the shape 'this' Value. /// - virtual const NDShape& Shape() const { - return m_data->Shape(); - } + virtual const NDShape& Shape() const { return m_data->Shape(); } /// /// Returns a boolean indicating if 'this' Value contains data in sparse storage format. /// - bool IsSparse() const - { - return (GetStorageFormat() != StorageFormat::Dense); - } + bool IsSparse() const { return (GetStorageFormat() != StorageFormat::Dense); } /// /// Returns a boolean indicating if 'this' Value is read-only. /// - virtual bool IsReadOnly() const { - return m_data->IsReadOnly(); - } + virtual bool IsReadOnly() const { return m_data->IsReadOnly(); } /// /// Returns the number of masked/invalid values /// - virtual size_t MaskedCount() const - { - return m_mask ? m_mask->MaskedCount() : 0; - } + virtual size_t MaskedCount() const { return m_mask ? m_mask->MaskedCount() : 0; } /// /// Returns the NDArrayView object corresponding to the data contents of 'this value object. @@ -2144,10 +2128,7 @@ namespace CNTK /// /// Creates a new Value with newly allocated storage on the same device as 'this' Value and copies 'this' Value's contents into the newly allocated Value. /// - ValuePtr DeepClone() const - { - return DeepClone(IsReadOnly()); - } + ValuePtr DeepClone() const { return DeepClone(IsReadOnly()); } /// /// Creates a new Value which is an alias of 'this' Value. @@ -2166,22 +2147,12 @@ namespace CNTK /// The Value should have the same tensor shape as outputVariable. /// template - void CopyTo(const Variable& outputVariable, std::vector>& sequences) + void CopyVariableValueTo(const Variable& outputVariable, std::vector>& sequences) { - std::vector seqLens; - CopyTo(outputVariable, sequences, seqLens, true); - } - - /// - /// Same as above, except if isResizable is false, the sequence buffer will not be resized. Instead, a runtime error is thrown. - /// In addition, on return, sequenceLengths contains the length of each sequence in sequences. - /// The sequenceLengths will be resized if necessary, even isResizable is false. - /// - template - void CopyTo(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable = true) - { - ResizeOutputBuffer(outputVariable, outputVariable.Shape().TotalSize(), sequences, sequenceLengths, isResizable); - CopyToVector(outputVariable, sequences, sequenceLengths); + if (outputVariable.GetDataType() != GetDataType()) + InvalidArgument("The outputVariable has a different data type than the Value object."); + ResizeOutputBuffer(outputVariable, sequences); + CopyVariableValueToVector(outputVariable, sequences); } /// @@ -2190,29 +2161,20 @@ namespace CNTK /// The sequence buffer will be resized if ncessary. /// The Value should have the same tensor shape as outputVariable. /// - void CopyTo(const Variable& outputVariable, std::vector>& sequences) + void CopyVariableValueTo(const Variable& outputVariable, std::vector>& sequences) { - std::vector seqLens; - CopyTo(outputVariable, sequences, seqLens, true); - } - - /// - /// Same as buffer, except if isResizable is false, the sequence buffer will not be resized. Instead, a runtime error is thrown. - /// In addition, sequenceLengths contains the length of each sequence in the sequence buffer. - /// The sequenceLengths will be resized if necessary, even isResizable is false. - /// - void CopyTo(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable = true) - { - // For OneHot vector, only 1 value is needed for a sample. - ResizeOutputBuffer(outputVariable, 1, sequences, sequenceLengths, isResizable); auto dataType = GetDataType(); + if (outputVariable.GetDataType() != dataType) + InvalidArgument("The outputVariable has a different data type than the Value object."); + + ResizeOutputBuffer(outputVariable, sequences); if (dataType == DataType::Float) { - CopyToVector(outputVariable, sequences, sequenceLengths); + CopyVariableValueToVector(outputVariable, sequences); } else if (dataType == DataType::Double) { - CopyToVector(outputVariable, sequences, sequenceLengths); + CopyVariableValueToVector(outputVariable, sequences); } } @@ -2222,29 +2184,63 @@ namespace CNTK CNTK_API static ValuePtr Create(const NDShape& sampleShape, const std::vector& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly, bool createNewCopy); + /// + /// Copy the data stored in 'this' Value object to the buffer 'sequences' as a collection of variable length sequences. + /// The output data is in the dense format. + /// Assumption: The 'sequences' buffer has been resized to match the number of sequences and the length of each sequence stored in the Value object. + /// The resizing is done by ResizeOutputBuffer() and needs to be done on the heap of the caller. + /// template - CNTK_API void CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); + void CopyVariableValueToVector(const Variable& outputVariable, std::vector>& sequences); + /// + /// Copy the data stored in 'this' Value object to the buffer 'sequences' as a collection of variable length sequences. + /// The output data is in the one-hot format. + /// The resizing is done by ResizeOutputBuffer() and needs to be done on the heap of the caller. + /// Assumption: The 'sequences' buffer has been resized to match the number of sequences and the length of each sequence stored in the Value object. + /// template - CNTK_API void CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); + void CopyVariableValueToVector(const Variable& outputVariable, std::vector>& sequences); template - void CopyToImpl(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); + void CopyVariableValueToImpl(const Variable& outputVariable, std::vector>& sequences); virtual std::pair GetSequenceAndBatchLength(const Variable& outputVariable); + /// + /// Resize the 'sequences' buffer if needed. + /// It should be kept in the header file, as the memory should be allocated at the caller side, not the CNTKLibarary.dll side. + /// outputVariable defines tensor, the sequence axis and the batch axis. + /// The 'sequences' is the output buffer which is used to store data from 'this' Value. On return, its size and the size of its each element are adjusted + /// to match the number of sequences and the length of each sequence stored in the Value object. + /// template - void ResizeOutputBuffer(const Variable& outputVariable, size_t sampleSize, std::vector>& sequences, std::vector& sequenceLengths, bool isResizable) + void ResizeOutputBuffer(const Variable& outputVariable, std::vector>& sequences) { + auto shape = outputVariable.Shape(); + if (shape == NDShape::Unknown || shape.HasInferredDimension()) + RuntimeError("It is not supported that the outputVariable has a unknown shape or inferred dimension."); + size_t numOfSequences; size_t maxSequenceLen; std::tie(maxSequenceLen, numOfSequences) = GetSequenceAndBatchLength(outputVariable); - // resize the sequnce length buffer to reflect the number of sequences in output. - if (sequenceLengths.size() < numOfSequences) - sequenceLengths.resize(numOfSequences); + // Calculate the number of elements is needed to represent a sample in output buffer. + // For dense output, it is the total size of the shape. + // For one-hot output, only 1 index is needed to represent the sample. + size_t outputSizeOfSample; + if (std::is_same::value) + { + outputSizeOfSample = 1; + } + else + { + outputSizeOfSample = shape.TotalSize(); + } + + // resize the output buffer size to reflect the number of sequences in output. + sequences.resize(numOfSequences); - // Check whether the output buffer is sufficient and allocate new space if isResizable is true. const MaskKind* maskData = nullptr; NDMaskPtr cpuMask = nullptr; if (Mask() != nullptr) @@ -2253,15 +2249,8 @@ namespace CNTK maskData = cpuMask->DataBuffer(); } + // Check whether each sequence has enough space allocated and resize if necessary. size_t sampleCount = 0, seqStart; - if (sequences.size() < numOfSequences) - { - if (isResizable) - sequences.resize(numOfSequences); - else - RuntimeError("The size of output buffer is too small"); - } - for (auto seqIndex = 0; seqIndex < numOfSequences; seqIndex++) { if (maskData == nullptr) @@ -2271,40 +2260,31 @@ namespace CNTK else { seqStart = seqIndex * maxSequenceLen; + // The assumption here is that a sequence always start at 0 with SequenceBegin (as no SequenceStart flag is returned) or + // with Invalid (empty sequence), and ends at the first invalid mask. if (maskData[seqStart] == MaskKind::Invalid) - RuntimeError("No leading invalid mask is allowed for a sequence."); - // The assumption here is that a sequence always start at 0 (no invalid mark at the beginning), - // and ends at the first invalid mask. - bool isEnd = false; - sampleCount = 0; - for (size_t i = 0; i < maxSequenceLen; i++) { - if (!isEnd) + // The sequence is empty. + sampleCount = 0; + } + else + { + if (maskData[seqStart] != MaskKind::SequenceBegin) + RuntimeError("Currently, only sequence starting with SequenceBegin is supported."); + sampleCount = 1; + while (sampleCount < maxSequenceLen) { - if (maskData[seqStart + i] != MaskKind::Invalid) + if (maskData[seqStart + sampleCount] == MaskKind::Valid) sampleCount++; else - isEnd = true; + break; } - // Todo: remove the check below if the assumption above is confirmed. - if (isEnd && (maskData[seqStart + i] != MaskKind::Invalid)) - RuntimeError("Invalid sequence."); } - assert(isEnd || (maskData[seqStart + maxSequenceLen - 1] != MaskKind::Invalid && sampleCount == maxSequenceLen)); } - sequenceLengths[seqIndex] = sampleCount; - if (sequences[seqIndex].size() < sampleCount * sampleSize) - { - if (isResizable) - sequences[seqIndex].resize(sampleCount * sampleSize); - else - RuntimeError("The size of output buffer is too small"); - } + // resize the sequence buffer to reflect the actual length in output. + sequences[seqIndex].resize(sampleCount * outputSizeOfSample); } - - for (size_t i = numOfSequences; i < sequenceLengths.size(); i++) - sequenceLengths[i] = 0; } // Disallow copy and move construction and assignment diff --git a/Source/CNTKv2LibraryDll/Value.cpp b/Source/CNTKv2LibraryDll/Value.cpp index 0f0d84c5bcf5..7f6acca14f0a 100644 --- a/Source/CNTKv2LibraryDll/Value.cpp +++ b/Source/CNTKv2LibraryDll/Value.cpp @@ -360,35 +360,33 @@ namespace CNTK } template - void DirectCopy(const ElementType *source, size_t sampleCount, size_t sampleSize, std::vector& dest); + void DirectCopy(const ElementType *source, size_t elementCount, std::vector& dest); template - void CopyDenseToOneHot(const ElementType *source, size_t sampleCount, size_t sampleSize, std::vector& dest); + void CopyDenseToOneHot(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest); template - void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths) + void Value::CopyVariableValueToVector(const Variable& outputVariable, std::vector>& sequences) { // Check the data type matches if (AsDataType() != GetDataType()) InvalidArgument("The specified ElementType %s does not match the DataType %s", typeid(ElementType).name(), DataTypeName(GetDataType())); - CopyToImpl(outputVariable, sequences, sequenceLengths); + CopyVariableValueToImpl(outputVariable, sequences); } template - void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths) + void Value::CopyVariableValueToVector(const Variable& outputVariable, std::vector>& sequences) { if (outputVariable.Shape()[0] != outputVariable.Shape().TotalSize()) { - InvalidArgument("The sample variable's leading axis dimensionality must equal the total size of the variable for sparse data."); + InvalidArgument("The outputVariable's leading axis dimensionality must equal the total size of the variable for sparse data."); } - CopyToImpl(outputVariable, sequences, sequenceLengths); + CopyVariableValueToImpl(outputVariable, sequences); } template - void Value::CopyToImpl(const Variable& outputVariable, - std::vector>& sequences, - std::vector& sequenceLengths) + void Value::CopyVariableValueToImpl(const Variable& outputVariable, std::vector>& sequences) { // PackedValue should be automatically unpacked when accessing Data() and Mask(). size_t numOfSequences; @@ -403,13 +401,13 @@ namespace CNTK NDArrayViewPtr cpuArrayView; if (Device().Type() != DeviceKind::CPU) { - // Todo: leverage sparse if the original NDArrayView is in spase. + // TODO: leverage sparse if the original NDArrayView is in spase. cpuArrayView = MakeSharedObject(GetDataType(), Shape(), DeviceDescriptor::CPUDevice()); cpuArrayView->CopyFrom(*Data()); } else { - // Todo: direct process sparse data without copy + // TODO: direct process sparse data without copy if (GetStorageFormat() != StorageFormat::Dense) { cpuArrayView = MakeSharedObject(GetDataType(), Shape(), DeviceDescriptor::CPUDevice()); @@ -429,37 +427,43 @@ namespace CNTK // The assumption here is that a sequence always start at 0 (no invaid mark at the beginning), // and ends at the first invalid mask. - // Therefore, no need to check NDMask again. The SequenceLengths already contains the correct number of samples. + // Therefore, no need to check NDMask again. + // And the sequences has been resized to match the number of sequences and the length of each sequence in the Value object. - // Todo: if function pointer or lambda could support template, switch to use them. + // TODO: if function pointer or lambda could support template, switch to use them. if (std::is_same::value) { - CopyDenseToOneHot(valueData + seqStart * sampleSize, sequenceLengths[seqIndex], sampleSize, sequences[seqIndex]); + // If the output is of the one-hot vector format, each value in sequences[seqIndex] is an index which represents a sample of sampleSize elements. + CopyDenseToOneHot(valueData + seqStart * sampleSize, sequences[seqIndex].size(), sampleSize, sequences[seqIndex]); } else { - DirectCopy(valueData + seqStart * sampleSize, sequenceLengths[seqIndex], sampleSize, sequences[seqIndex]); + // If the output is of the dense format, each value in sequences[seqIndex] represents an element of a sample. + DirectCopy(valueData + seqStart * sampleSize, sequences[seqIndex].size(), sequences[seqIndex]); } } } std::pair Value::GetSequenceAndBatchLength(const Variable& outputVariable) { - size_t sampleRank = outputVariable.Shape().Rank(); + size_t varRank = outputVariable.Shape().Rank(); size_t maxSequenceLength = 1; size_t numSequences = 1; - if (Shape().Rank() < sampleRank) + if (Shape().Rank() < varRank) RuntimeError("The Value'rank should be greater than or equal to the variable's rank."); - size_t maskRank = Shape().Rank() - sampleRank; - if (outputVariable.Shape() != Shape().SubShape(0, sampleRank)) - RuntimeError("The shape of the outputVariable does not match the value shape."); + size_t maskRank = Shape().Rank() - varRank; + if (outputVariable.Shape() != Shape().SubShape(0, varRank)) + RuntimeError("The shape of the outputVariable does not match the Value shape."); + + if (outputVariable.DynamicAxes().size() > 2) + LogicError("More than 2 dynamic axis for a variable is currently unsupported"); if (maskRank > 2) - RuntimeError("Only 2 dynamic axes are supported now."); + LogicError("Value rank which is larger than the output variable rank by more than 2 dynamic axes is currently unsupported."); - std::tie(maxSequenceLength, numSequences) = GetNumTimeStepsAndSequences(Shape().SubShape(sampleRank), outputVariable.DynamicAxes().size()); + std::tie(maxSequenceLength, numSequences) = GetNumTimeStepsAndSequences(Shape().SubShape(varRank), outputVariable.DynamicAxes().size()); return std::pair(maxSequenceLength, numSequences); } @@ -499,15 +503,15 @@ namespace CNTK } template - void DirectCopy(const ElementType *source, const size_t sampleCount, const size_t sampleSize, std::vector& dest) + void DirectCopy(const ElementType *source, const size_t elementCount, std::vector& dest) { if (!std::is_same::value) RuntimeError("Source and destination must be the same data type."); DestType *destData = dest.data(); - if (sampleCount * sampleSize > dest.size()) + if (elementCount > dest.size()) RuntimeError("The output buffer is too small."); - std::copy(source, source + sampleCount * sampleSize, reinterpret_cast(destData)); + std::copy(source, source + elementCount, reinterpret_cast(destData)); } template @@ -551,8 +555,8 @@ namespace CNTK template /*static*/ CNTK_API ValuePtr Value::Create(const NDShape& sampleShape, const std::vector>& sequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); template /*static*/ CNTK_API ValuePtr Value::Create(size_t vocabSize, const std::vector>& oneHotSequences, const std::vector& sequenceStartFlags, const DeviceDescriptor& device, bool readOnly/* = false*/); - template CNTK_API void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequencesLens); - template CNTK_API void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequencesLens); - template CNTK_API void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); - template CNTK_API void Value::CopyToVector(const Variable& outputVariable, std::vector>& sequences, std::vector& sequenceLengths); + template CNTK_API void Value::CopyVariableValueToVector(const Variable& outputVariable, std::vector>& sequences); + template CNTK_API void Value::CopyVariableValueToVector(const Variable& outputVariable, std::vector>& sequences); + template CNTK_API void Value::CopyVariableValueToVector(const Variable& outputVariable, std::vector>& sequences); + template CNTK_API void Value::CopyVariableValueToVector(const Variable& outputVariable, std::vector>& sequences); } diff --git a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp index 2713a2b778b8..ff568f4d586a 100644 --- a/Tests/UnitTests/V2LibraryTests/ValueTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/ValueTests.cpp @@ -62,7 +62,7 @@ void CheckValue(const ValuePtr testValue, const NDShape& sampleShape, const vect for (size_t seq = 0; seq < seqLenList.size(); seq++) { - // Todo: need to check NDMask is also correct. It is partly done in CopyTo tests. + // TODO: need to check NDMask is also correct. It is partly done in CopyTo tests. size_t seqLen = seqLenList[seq]; for (size_t sIndex = 0; sIndex < seqLen * sampleSize; sIndex++, oIndex++) { @@ -271,39 +271,18 @@ void ValueCreationOneHotWithNDMaskTest(const DeviceDescriptor device, bool readO } template -void CheckCopyToOutput(const size_t sampleSize, const std::vector>& expected, const std::vector>& actual, const std::vector& actualSeqLens) +void CheckCopyToOutput(const std::vector>& expected, const std::vector>& actual) { - bool useSeqLens; - if (actualSeqLens.size() != 0) - { - useSeqLens = true; - if (actualSeqLens.size() < expected.size()) - ReportFailure("The actualSeqLens size does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), actualSeqLens.size()); - else - { - for (size_t i = expected.size(); i < actualSeqLens.size(); i++) - if (actualSeqLens[i] != 0) - ReportFailure("The actualSeqLens contains invalid data."); - } - - if (actual.size() < expected.size()) - ReportFailure("The number of sequences does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), actual.size()); - } - else - { - useSeqLens = false; - if (expected.size() != actual.size()) - ReportFailure("The number of sequences does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), actual.size()); - } + if (expected.size() != actual.size()) + ReportFailure("The number of sequences does not match. expected: %" PRIu64 " actual: %" PRIu64 "\n", expected.size(), actual.size()); for (size_t i = 0; i < expected.size(); i++) { - auto len = useSeqLens ? actualSeqLens[i] * sampleSize : actual[i].size(); - if ((actual[i].size() < len) || (expected[i].size() != len)) + if (expected[i].size() != actual[i].size()) { ReportFailure("Seq %lu does not match.\n", static_cast(i)); } - for (size_t j = 0; j < len; j++) + for (size_t j = 0; j < expected[i].size(); j++) { if (expected[i][j] != actual[i][j]) { @@ -313,13 +292,6 @@ void CheckCopyToOutput(const size_t sampleSize, const std::vector -void CheckCopyToOutput(const size_t sampleSize, const std::vector>& expected, const std::vector>& actual) -{ - std::vector actualSeqLens(0); - CheckCopyToOutput(sampleSize, expected, actual, actualSeqLens); -} - template Variable CreateVariable(NDShape shape, int numOfDynamicAxes) { @@ -347,48 +319,38 @@ Variable CreateVariable(NDShape shape, int numOfDynamicAxes) } template -void TestResizeDense(const Variable& sampleVariable, std::vector& expectedSeqLens, std::vector>& output, std::vector& actualSeqLens, const DeviceDescriptor& device) +void TestDenseSequences(const Variable& sampleVariable, std::vector& expectedSeqLens, std::vector>& output, const DeviceDescriptor& device) { auto input = GenerateSequences(expectedSeqLens, sampleVariable.Shape()); auto val = Value::Create(sampleVariable.Shape(), input, device); - // The sequence axis is too small - VerifyException([&val, &sampleVariable, &output, &actualSeqLens]() { - val->CopyTo(sampleVariable, output, actualSeqLens, false); - }, "The exception 'output buffer is too small.' not thrown."); - - val->CopyTo(sampleVariable, output, actualSeqLens); - CheckCopyToOutput(sampleVariable.Shape().TotalSize(), input, output, actualSeqLens); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); } template -void TestResizeOneHot(const Variable& sampleVariable, std::vector& expectedSeqLens, std::vector>& output, std::vector& actualSeqLens, const DeviceDescriptor& device) +void TestOneHotSequences(const Variable& sampleVariable, std::vector& expectedSeqLens, std::vector>& output, const DeviceDescriptor& device) { auto input = GenerateOneHotSequences(expectedSeqLens, sampleVariable.Shape().TotalSize()); auto val = Value::Create(sampleVariable.Shape().TotalSize(), input, device); - // The sequence axis is too small - VerifyException([&val, &sampleVariable, &output, &actualSeqLens]() { - val->CopyTo(sampleVariable, output, actualSeqLens, false); - }, "The exception 'output buffer is too small.' not thrown."); - - val->CopyTo(sampleVariable, output, actualSeqLens); - CheckCopyToOutput(1, input, output, actualSeqLens); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); } template void ValueCopyToDenseTest(const DeviceDescriptor& device) { NDShape sampleShape{{2, 3}}; - auto sampleSize = sampleShape.TotalSize(); std::vector> input; std::vector> output; std::vector expectedSeqLens; - std::vector actualSeqLens; - //Todo: add tests sparse to dense. - // Check single sample. No dynamic axis for the sampleVariable - auto sampleVariable = CreateVariable(sampleShape, 0); + //TODO: add tests sparse to dense. + + // Check single sample. + // No dynamic axis for the sampleVariable + auto sampleVariable = CreateVariable(sampleShape, 0); size_t batchCount = 1; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) @@ -396,22 +358,22 @@ void ValueCopyToDenseTest(const DeviceDescriptor& device) input = GenerateSequences(expectedSeqLens, sampleShape); auto val = Value::Create(sampleShape, input, device); - val->CopyTo(sampleVariable, output); - CheckCopyToOutput(sampleSize, input, output); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); // 1 dynamic axis (as batch) for the sampleVariable - sampleVariable = CreateVariable(sampleShape, 1); - val->CopyTo(sampleVariable, output); - CheckCopyToOutput(sampleSize, input, output); + sampleVariable = CreateVariable(sampleShape, 1); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); // 2 dynamic axes for the sampleVariable - sampleVariable = CreateVariable(sampleShape, 2); - val->CopyTo(sampleVariable, output); - CheckCopyToOutput(sampleSize, input, output); + sampleVariable = CreateVariable(sampleShape, 2); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); - // Check batch of sample. + // Check batch of samples. // 1 dynamic axis (as batch) for the sampleVariable - sampleVariable = CreateVariable(sampleShape, 1); + sampleVariable = CreateVariable(sampleShape, 1); batchCount = 2; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) @@ -419,55 +381,58 @@ void ValueCopyToDenseTest(const DeviceDescriptor& device) input = GenerateSequences(expectedSeqLens, sampleVariable.Shape()); val = Value::Create(sampleVariable.Shape(), input, device); - // The sequence axis is too small - // Not using TestResizeDense as the input value is needed for the next test. - VerifyException([&val, &sampleVariable, &output, &actualSeqLens]() { - val->CopyTo(sampleVariable, output, actualSeqLens, false); - }, "The exception 'output buffer is too small.' not thrown."); - - val->CopyTo(sampleVariable, output, actualSeqLens); - CheckCopyToOutput(sampleVariable.Shape().TotalSize(), input, output, actualSeqLens); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); // 2 dynamic axes for the sampleVariable - sampleVariable = CreateVariable(sampleShape, 2); - val->CopyTo(sampleVariable, output); - CheckCopyToOutput(sampleSize, input, output); + sampleVariable = CreateVariable(sampleShape, 2); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); - // Check sequence of sample, but single batch + // Check sequence of samples, but single batch // The variable should have 2 dynamic axes. - sampleVariable = CreateVariable(sampleShape, 2); + sampleVariable = CreateVariable(sampleShape, 2); size_t sampleCount = 4; batchCount = 1; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCount); - TestResizeDense(sampleVariable, expectedSeqLens, output, actualSeqLens, device); + TestDenseSequences(sampleVariable, expectedSeqLens, output, device); - // Check batch of sequence of the same length, no mask needed. + // Check batch of sequences of the same length, no mask needed. batchCount = 4; sampleCount = 3; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCount); - TestResizeDense(sampleVariable, expectedSeqLens, output, actualSeqLens, device); + TestDenseSequences(sampleVariable, expectedSeqLens, output, device); - // Check batch of sequecnes with different length, mask needed. - std::vector sampleCountList {6, 9, 2}; + // Check batch of sequecnes with different lengths, mask needed. + // The length of one sequence is 1. + std::vector sampleCountList = {6, 1, 2}; batchCount = sampleCountList.size(); expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCountList[i]); - TestResizeDense(sampleVariable, expectedSeqLens, output, actualSeqLens, device); + TestDenseSequences(sampleVariable, expectedSeqLens, output, device); - // More batches and sequences, need resize + // Check batch of sequecnes with different lengths, mask needed. + sampleCountList = {6, 9, 2}; + batchCount = sampleCountList.size(); + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCountList[i]); + TestDenseSequences(sampleVariable, expectedSeqLens, output, device); + + // More sequences in a batch, need resize sampleCountList = {6, 12, 2, 1, 5, 3, 4}; batchCount = sampleCountList.size(); expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCountList[i]); - TestResizeDense(sampleVariable, expectedSeqLens, output, actualSeqLens, device); + TestDenseSequences(sampleVariable, expectedSeqLens, output, device); - // Random batch and sequence + // Random batch and sequences int testRun = 4; size_t maxNumOfSequences = 100; size_t maxSequenceLen = 100; @@ -482,8 +447,8 @@ void ValueCopyToDenseTest(const DeviceDescriptor& device) input = GenerateSequences(expectedSeqLens, sampleShape); val = Value::Create(sampleShape, input, device); - val->CopyTo(sampleVariable, output, actualSeqLens); - CheckCopyToOutput(sampleSize, input, output, actualSeqLens); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput( input, output); } } @@ -495,11 +460,11 @@ void ValueCopyToOneHotTest(const DeviceDescriptor& device) std::vector> input; std::vector> output; std::vector expectedSeqLens; - std::vector actualSeqLens; - // Todo: add tests dense to sparse + // TODO: add tests dense to sparse // Check single sample. - auto sampleVariable = CreateVariable(sampleShape, 0); + // No dynamic axis for the sampleVariable. + auto sampleVariable = CreateVariable(sampleShape, 0); size_t batchCount = 1; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) @@ -507,22 +472,22 @@ void ValueCopyToOneHotTest(const DeviceDescriptor& device) input = GenerateOneHotSequences(expectedSeqLens, dim); auto val = Value::Create(dim, input, device); - val->CopyTo(sampleVariable, output); - CheckCopyToOutput(1, input, output); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); // 1 dynamic axis (as batch) for the sampleVariable - sampleVariable = CreateVariable(sampleShape, 1); - val->CopyTo(sampleVariable, output); - CheckCopyToOutput(1, input, output); + sampleVariable = CreateVariable(sampleShape, 1); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); // 2 dynamic axes for the sampleVariable - sampleVariable = CreateVariable(sampleShape, 2); - val->CopyTo(sampleVariable, output); - CheckCopyToOutput(1, input, output); + sampleVariable = CreateVariable(sampleShape, 2); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); - // Check batch of sample. + // Check batch of samples. // 1 dynamic axis (as batch) for the sampleVariable - sampleVariable = CreateVariable(sampleShape, 1); + sampleVariable = CreateVariable(sampleShape, 1); batchCount = 2; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) @@ -530,55 +495,58 @@ void ValueCopyToOneHotTest(const DeviceDescriptor& device) input = GenerateOneHotSequences(expectedSeqLens, sampleVariable.Shape().TotalSize()); val = Value::Create(sampleVariable.Shape().TotalSize(), input, device); - // The sequence axis is too small - // Not using TestResize() as the input is needed for the next test. - VerifyException([&val, &sampleVariable, &output, &actualSeqLens]() { - val->CopyTo(sampleVariable, output, actualSeqLens, false); - }, "The exception 'output buffer is too small.' not thrown."); - - val->CopyTo(sampleVariable, output, actualSeqLens); - CheckCopyToOutput(1, input, output, actualSeqLens); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); // 2 dynamic axes for the sampleVariable - sampleVariable = CreateVariable(sampleShape, 2); - val->CopyTo(sampleVariable, output); - CheckCopyToOutput(1, input, output); + sampleVariable = CreateVariable(sampleShape, 2); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); - // Check sequence of sample, but single batch + // Check sequence of samples, but single batch // The variable should have 2 dynamic axes. - sampleVariable = CreateVariable(sampleShape, 2); + sampleVariable = CreateVariable(sampleShape, 2); size_t sampleCount = 4; batchCount = 1; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCount); - TestResizeOneHot(sampleVariable, expectedSeqLens, output, actualSeqLens, device); + TestOneHotSequences(sampleVariable, expectedSeqLens, output, device); - // Check batch of sequence of the same length, no mask needed. + // Check batch of sequences of the same length, no mask needed. batchCount = 4; sampleCount = 3; expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCount); - TestResizeOneHot(sampleVariable, expectedSeqLens, output, actualSeqLens, device); + TestOneHotSequences(sampleVariable, expectedSeqLens, output, device); + + // Check batch of sequecnes with different lengths, mask needed. + // The length of one sequence is 1. + std::vector sampleCountList = {6, 1, 2}; + batchCount = sampleCountList.size(); + expectedSeqLens.clear(); + for (size_t i = 0; i < batchCount; i++) + expectedSeqLens.push_back(sampleCountList[i]); + TestOneHotSequences(sampleVariable, expectedSeqLens, output, device); - // Check batch of sequecnes with different length, mask needed. - std::vector sampleCountList{6, 9, 2}; + // Check batch of sequecnes with different lengths, mask needed. + sampleCountList = {6, 9, 2}; batchCount = sampleCountList.size(); expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCountList[i]); - TestResizeOneHot(sampleVariable, expectedSeqLens, output, actualSeqLens, device); + TestOneHotSequences(sampleVariable, expectedSeqLens, output, device); - // More batches and sequences, resize required + // More sequences in a batch, resize required sampleCountList = {6, 12, 2, 1, 5, 3, 4}; batchCount = sampleCountList.size(); expectedSeqLens.clear(); for (size_t i = 0; i < batchCount; i++) expectedSeqLens.push_back(sampleCountList[i]); - TestResizeOneHot(sampleVariable, expectedSeqLens, output, actualSeqLens, device); + TestOneHotSequences(sampleVariable, expectedSeqLens, output, device); - // Random batch and sequence + // Random batch and sequences int testRun = 4; size_t maxNumOfSequences = 100; size_t maxSequenceLen = 100; @@ -593,11 +561,60 @@ void ValueCopyToOneHotTest(const DeviceDescriptor& device) input = GenerateOneHotSequences(expectedSeqLens, dim); val = Value::Create(dim, input, device); - val->CopyTo(sampleVariable, output, actualSeqLens); - CheckCopyToOutput(1, input, output, actualSeqLens); + val->CopyVariableValueTo(sampleVariable, output); + CheckCopyToOutput(input, output); } } +void ValueCopyToExceptionsTest(const DeviceDescriptor& device) +{ + std::vector expectedSeqLens = {1}; + std::vector> input; + std::vector> output; + std::vector> outputInDouble; + std::vector> outputInOneHot; + NDShape sampleShape{{2, 3}}; + NDShape sampleOneHotShape{{100}}; + + input = GenerateSequences(expectedSeqLens, sampleShape); + auto val = Value::Create(sampleShape, input, device); + + // Test variable with unknown shape + auto sampleVariable = CreateVariable(NDShape::Unknown, 0); + VerifyException([&val, &sampleVariable, &output]() { + val->CopyVariableValueTo(sampleVariable, output); + }, "The expected exception has not been caugth: It is not supported that the outputVariable has a unknown shape or inferred dimension."); + + // Test variable having shape with InferredDimentsion. + sampleVariable = CreateVariable(NDShape(2), 0); + VerifyException([&val, &sampleVariable, &output]() { + val->CopyVariableValueTo(sampleVariable, output); + }, "The expected exception has not been caugth: It is not supported that the outputVariable has a unknown shape or inferred dimension."); + + // Test variable having incorrect data type. + sampleVariable = CreateVariable(sampleShape, 0); + VerifyException([&val, &sampleVariable, &output]() { + val->CopyVariableValueTo(sampleVariable, output); + }, "The expected exception has not been caugth: The outputVariable has a different data type than the Value object."); + + sampleVariable = CreateVariable(sampleOneHotShape, 0); + VerifyException([&val, &sampleVariable, &outputInOneHot]() { + val->CopyVariableValueTo(sampleVariable, outputInOneHot); + }, "The expected exception has not been caugth: The outputVariable has a different data type than the Value object."); + + // Test output buffer having incorrect data type. + sampleVariable = CreateVariable(sampleShape, 0); + VerifyException([&val, &sampleVariable, &outputInDouble]() { + val->CopyVariableValueTo(sampleVariable, outputInDouble); + }, "The expected exception has not been caugth: The specified ElementType Double does not match the DataType Float"); + + // Test the first axis when using one-hot format. + VerifyException([&val, &sampleVariable, &outputInOneHot]() { + val->CopyVariableValueTo(sampleVariable, outputInOneHot); + }, "The expected exception has not been caugth: The outputVariable's leading axis dimensionality must equal the total size of the variable for sparse data."); +} + + void TestSettingParameterValuesManually(const DeviceDescriptor& device) { auto v1_1 = MakeSharedObject(0.5, NDShape({ 2, 2 }), device); @@ -698,4 +715,6 @@ void ValueTests() ValueCopyToOneHotTest(DeviceDescriptor::GPUDevice(0)); ValueCopyToOneHotTest(DeviceDescriptor::GPUDevice(0)); } + + ValueCopyToExceptionsTest(DeviceDescriptor::CPUDevice()); } From 09daade4f080d930af51f78508769bd589d25bc9 Mon Sep 17 00:00:00 2001 From: PengchengHe Date: Tue, 3 Jan 2017 15:28:17 +0000 Subject: [PATCH 039/120] Add ReasoNet model as example --- Examples/ReasoNet/__init__.py | 0 Examples/ReasoNet/__main__.py | 25 ++ Examples/ReasoNet/asr.py | 256 +++++++++++++++++ Examples/ReasoNet/model.py | 260 ++++++++++++++++++ Examples/ReasoNet/test.idx | 5 + Examples/ReasoNet/test.py | 159 +++++++++++ Examples/ReasoNet/tests/testReasoNetModel.py | 64 +++++ .../ReasoNet/tests/testTextConvolution.py | 13 + Examples/ReasoNet/wordvocab.py | 200 ++++++++++++++ Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 2 +- Source/CNTKv2LibraryDll/CompositeFunction.cpp | 36 ++- bindings/python/cntk/layers.py | 6 +- bindings/python/setup.py | 2 +- 13 files changed, 1024 insertions(+), 4 deletions(-) create mode 100644 Examples/ReasoNet/__init__.py create mode 100644 Examples/ReasoNet/__main__.py create mode 100644 Examples/ReasoNet/asr.py create mode 100644 Examples/ReasoNet/model.py create mode 100644 Examples/ReasoNet/test.idx create mode 100644 Examples/ReasoNet/test.py create mode 100644 Examples/ReasoNet/tests/testReasoNetModel.py create mode 100644 Examples/ReasoNet/tests/testTextConvolution.py create mode 100644 Examples/ReasoNet/wordvocab.py diff --git a/Examples/ReasoNet/__init__.py b/Examples/ReasoNet/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/Examples/ReasoNet/__main__.py b/Examples/ReasoNet/__main__.py new file mode 100644 index 000000000000..fa1ace65e52d --- /dev/null +++ b/Examples/ReasoNet/__main__.py @@ -0,0 +1,25 @@ +import sys +import os +import argparse +from .wordvocab import Vocabulary + +def file_exists(src): + return (os.path.isfile(src) and os.path.exists(src)) + +parser = argparse.ArgumentParser() +parser.add_argument('-t', '--train-data', required=True, help="Path of the training data.") +parser.add_argument('-v', '--vocab', default="word_vocab.txt", help="Path of the word vocabulary data.") +parser.add_argument('-s', '--vocab-size', default=50000, type=int, help="Max size of the word vocabulary.") +parser.add_argument('-i', '--train-index', default="train.index.txt", help="The path of the featurized index file.") +args = parser.parse_args() +entity_vocab = None +word_vocab = None +if not file_exists(args.vocab): + entity_vocab, word_vocab = Vocabulary.build_bingvocab(args.train_data, args.vocab, args.vocab_size) +else: + print("Load") + entity_vocab, word_vocab = Vocabulary.load_bingvocab(args.vocab) + +if not file_exists(args.train_index): + print("Gen") + Vocabulary.build_bing_corpus_index(entity_vocab, word_vocab, args.train_data, args.train_index) diff --git a/Examples/ReasoNet/asr.py b/Examples/ReasoNet/asr.py new file mode 100644 index 000000000000..47392a1fa51c --- /dev/null +++ b/Examples/ReasoNet/asr.py @@ -0,0 +1,256 @@ +""" +ReasoNet model in CNTK +@author penhe@microsoft.com +""" +import sys +import os +import numpy as np +from cntk import Trainer, Axis, device, combine +from cntk.io import MinibatchSource, CTFDeserializer, StreamDef, StreamDefs, INFINITELY_REPEAT +from cntk.learner import momentum_sgd, momentum_as_time_constant_schedule, learning_rate_schedule, UnitType +from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ + parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, tanh, times_transpose, greater, cosine_distance, element_divide, element_select +from cntk.blocks import LSTM, Stabilizer, _get_current_default_options, _is_given, _initializer_for, _resolve_activation, _INFERRED, Parameter, Placeholder, Block, init_default_or_glorot_uniform +from cntk.layers import Recurrence, Convolution +from cntk.initializer import uniform, glorot_uniform +from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input +from cntk.utils.debughelpers import _name_node, _node_name, _node_description, _log_node + + +######################## +# variables and stuff # +######################## + +data_dir = "./data" +model_dir = "./models" + +# model dimensions +#vocab_dim = 40000 +#embed_dim = 200 +#hidden_dim = 256 +#src_max_len = 120 +#ctx_max_len = 1989 + +# stabilizer +stabilize = Stabilizer() + +def text_convolution(win_size, in_dim, out_dim): + #activation = _resolve_activation(activation) + output_channels_shape = _as_tuple(out_dim) + output_rank = len(output_channels_shape) + filter_shape = (win_size, in_dim) + filter_rank = len(filter_shape) + kernel_shape = _INFERRED + filter_shape # kernel := filter plus reductionDims + + # parameters bound to this Function + init_kernel = glorot_uniform(filter_rank=filter_rank, output_rank=1) + #init_kernel = _initializer_for(init, Record(filter_rank=filter_rank, output_rank=-1)) + # BUGBUG: It is very confusing that output_rank is negative, esp. since that means count from the start. Solution: add a flag + W = Parameter(output_channels_shape + kernel_shape, init=init_kernel, name='W') # (K, C, H, W) aka [ W x H x C x K ] + #w = np.reshape(np.array([[[[2, -1, 0, -1, 2],[1,1,2,-1,-1],[1,2,0,2,1]]]], dtype = np.float32), (1, 1, 3, 5)) + #W = constant(value=w) + #b = Parameter(output_channels_shape + (1,) * len(filter_shape), init=init_bias, name='b') if bias else None # (K, 1, 1) aka [ 1 x 1 x K ] + + # expression + x = Placeholder(name='convolution_arg') + # TODO: update the parameter order of convolution() to match the optional ones as in here? (options order matches Keras) + strides = (1, 1, in_dim) + + apply_x = convolution (W, x, + strides = _as_tuple(strides), + sharing = _as_tuple(True), + auto_padding = _as_tuple(False), + lower_pad = (0, win_size/2, 0), + upper_pad = (0, (win_size-1)/2, 0) + ) +# # TODO: can we rename auto_padding to pad? + #if bias: + # apply_x = apply_x + b + apply_x = apply_x >> sigmoid + return Block(apply_x, 'Convolution', Record(W=W)) + +def create_reader(path, randomize, size=INFINITELY_REPEAT): + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + answer = StreamDef(field='A', shape=vocab_dim, is_sparse=True) + )), randomize=randomize, epoch_size = size) + + ######################## +# define the model # +######################## + +def gru_cell(shape, init=init_default_or_glorot_uniform, name=''): # (x, (h,c)) + shape = _as_tuple(shape) + + if len(shape) != 1 : + raise ValueError("gru_cell: shape must be vectors (rank-1 tensors)") + + # determine stacking dimensions + cell_shape_stacked = shape * 2 # patched dims with stack_axis duplicated 4 times + + # parameters + Wz = Parameter(cell_shape_stacked, init = init, name='Wz') + Wr = Parameter(cell_shape_stacked, init = init, name='Wr') + Wh = Parameter(cell_shape_stacked, init = init, name='Wh') + Uz = Parameter( _INFERRED + shape, init = init, name = 'Uz') + Ur = Parameter( _INFERRED + shape, init = init, name = 'Ur') + Uh = Parameter( _INFERRED + shape, init = init, name = 'Uh') + + def create_s_placeholder(): + # we pass the known dimensions here, which makes dimension inference easier + return Placeholder(shape=shape, name='S') # (h, c) + + # parameters to model function + x = Placeholder(name='gru_block_arg') + prev_status = create_s_placeholder() + + # formula of model function + Sn_1 = prev_status + + z = sigmoid(times(x, Uz, name='x*Uz') + times(Sn_1, Wz, name='Sprev*Wz'), name='z') + r = sigmoid(times(x, Ur, name='x*Ur') + times(Sn_1, Wr, name='Sprev*Wr'), name='r') + h = tanh(times(x, Uh, name='x*Uh') + times(element_times(Sn_1, r, name='Sprev*r'), Wh), name='h') + s = plus(element_times((1-z), h, name='(1-z)*h'), element_times(z, Sn_1, name='z*SPrev'), name=name) + apply_x_s = combine([s]) + apply_x_s.create_placeholder = create_s_placeholder + return apply_x_s + +def bidirectionalLSTM(hidden_dim, x, splice_outputs=True): + fwd = Recurrence(LSTM(hidden_dim), go_backwards=False) (stabilize(x)) + bwd = Recurrence(LSTM(hidden_dim), go_backwards=True ) (stabilize(x)) + if splice_outputs: + # splice the outputs together + hc = splice((fwd, bwd)) + return hc + else: + # return both (in cases where we want the 'final' hidden status) + return (fwd, bwd) + +def bidirectional_gru(hidden_dim, x, splice_outputs=True, name=''): + fwd = Recurrence(gru_cell(hidden_dim), go_backwards=False) (stabilize(x)) + bwd = Recurrence(gru_cell(hidden_dim), go_backwards=True) (stabilize(x)) + if splice_outputs: + # splice the outputs together + hc = splice((fwd, bwd), name=name) + return hc + else: + # return both (in cases where we want the 'final' hidden status) + return (fwd, bwd) + +def broadcast_as(op, seq_op, name=''): + x=placeholder_variable(shape=op.shape, name='src_op') + s=placeholder_variable(shape=seq_op.shape, name='tgt_seq') + pd = sequence.scatter(x, sequence.is_first(s, name='isf_1'), name='sct') + pout = placeholder_variable(op.shape, dynamic_axes=seq_op.dynamic_axes, name='pout') + out = element_select(sequence.is_first(s, name='isf_2'), pd, past_value(pout, name='ptv'), name='br_sel') + rlt = out.replace_placeholders({pout:sanitize_input(out), x:sanitize_input(op), s:sanitize_input(seq_op)}) + return combine([rlt], name=name) + +def cosine_similarity(src, tgt, name=''): + src_br = broadcast_as(src, tgt, name='cos_br') + sim = cosine_distance(src_br, tgt, name=name) + return sim + +def project_cosine_sim(status, memory, dim, init = init_default_or_glorot_uniform, name=''): + cell_shape = (dim, dim) + Wi = Parameter(cell_shape, init = init, name='Wi') + Wm = Parameter(cell_shape, init = init, name='Wm') + weighted_status = times(status, Wi, name = 'project_status') + weighted_memory = times(memory, Wm, name = 'project_memory') + return cosine_similarity(status, memory, name=name) + +def project_dotprod_sim(status, memory, dim, init = init_default_or_glorot_uniform, name=''): + cell_shape = (dim, dim) + Wi = Parameter(cell_shape, init = init, name='Wi') + Wm = Parameter(cell_shape, init = init, name='Wm') + weighted_status = times(status, Wi, name = 'project_status') + weighted_memory = times(memory, Wm, name = 'project_memory') + return times_transpose(broadcast_as(weighted_status, weighted_memory), weighted_memory) + +def termination_gate(status, dim, init = init_default_or_glorot_uniform, name=''): + Wt = Parameter((dim, 1), init = init, name='Wt') + return sigmoid(times(status, Wt), name=name) + +def attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids,hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): + status = Placeholder(name='status', shape=hidden_dim) + context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim, name='context_attention') + query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim, name='query_attetion') + context_attention = sequence.reduce_sum(times(context_attention_weight, context_memory), name='C-Att') + query_attention = sequence.reduce_sum(times(query_attention_weight, query_memory), name='Q-Att') + attention = splice((query_attention, context_attention), name='att-sp') + gru = gru_cell((hidden_dim, ), name='status') + new_status = gru(attention, status).output + termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') + ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim, name='ans_attention') + answers = times(ans_attention, candidate_ids, name='answers') + return combine([answers, termination_prob, new_status], name='ReinforcementAttention') + +def attention_rlunit2(hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): + status = Placeholder(name='status') + context_memory = Placeholder(name='context_memory') + query_memory = Placeholder(name='query_memory') + candidate_memory = Placeholder(name='candidate_memory') + candidate_ids = Placeholder(name='candidate_ids') + context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim) + query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim) + context_attention = reduce_sum(element_times(context_attention_weight, context_memory), axis = 0) + query_attention = reduce_sum(element_times(query_attention_weight, query_memory), axis = 0) + attention = splice((query_attention, context_attention)) + gru = gru_cell((hidden_dim, ), name='status') + new_status = gru(attention, status).output + termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') + ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim) + answers = times(ans_attention, candidate_ids, name='ans2') + return combine([answers, termination_prob, new_status], name='ReinforcementAttention') + +def set_dynamic_axes(dynamic_axes, shape): + src = placeholder_variable(shape=shape, dynamic_axes = dynamic_axes) + outp = placeholder_variable(shape=shape, dynamic_axes = dynamic_axes) + +def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glorot_uniform): + # Query and Doc/Context/Paragraph inputs to the model + batch_axis = Axis.default_batch_axis() + query_seq_axis = Axis('sourceAxis') + context_seq_axis = Axis('contextAxis') + query_dynamic_axes = [batch_axis, query_seq_axis] + query_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=query_dynamic_axes, name='query') + context_dynamic_axes = [batch_axis, context_seq_axis] + context_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') + candidate_dynamic_axes = [batch_axis, context_seq_axis] + candidate_indicates = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') + candidate_filter = greater(candidate_indicates, 0) + candidate_ids = sequence.gather(context_raw, candidate_filter) + + # Query sequences + query_sequence = query_raw + # Doc/Context sequences + context_sequence = context_raw + # embedding + embed_dim = hidden_dim + embedding = parameter(shape=(vocab_dim, embed_dim), init=uniform(1)) + + query_embedding = times(query_sequence , embedding) + context_embedding = times(context_sequence, embedding) + + # get source and context representations + context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis + candidate_memory = sequence.gather(context_memory, candidate_filter, name='Candidate_Mem') + + qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis + query_memory = splice((qfwd, qbwd), name='Query_SP') + # get the source (aka 'query') representation + status = splice((sequence.last(qfwd), sequence.first(qbwd)), name='Init_Status') # get last fwd status and first bwd status + attention_rlu = attention_rlunit(context_memory, query_memory, context_memory.output, context_sequence, hidden_dim*2, vocab_dim, init) + # TODO: the candidate_memory and candidate_ids are not in the same dynamic axes + #attention_rlu = attention_rlunit(context_memory, query_memory, candidate_memory.output, candidate_ids.output, hidden_dim*2, vocab_dim, init) + arlus = attention_rlu(status) + return combine(arlus.outputs, name='ReasoNet') + +def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + entities = StreamDef(field='E', shape=1, is_sparse=False), + label = StreamDef(field='L', shape=1, is_sparse=False) + )), randomize=randomize, epoch_size = size) diff --git a/Examples/ReasoNet/model.py b/Examples/ReasoNet/model.py new file mode 100644 index 000000000000..b0eb2eee0062 --- /dev/null +++ b/Examples/ReasoNet/model.py @@ -0,0 +1,260 @@ +""" +ReasoNet model in CNTK +@author penhe@microsoft.com +""" +import sys +import os +import numpy as np +from cntk import Trainer, Axis, device, combine +from cntk.io import MinibatchSource, CTFDeserializer, StreamDef, StreamDefs, INFINITELY_REPEAT +from cntk.learner import momentum_sgd, momentum_as_time_constant_schedule, learning_rate_schedule, UnitType +from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ + parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, tanh, times_transpose, greater, cosine_distance, element_divide, element_select +from cntk.blocks import LSTM, Stabilizer, _get_current_default_options, _is_given, _initializer_for, _resolve_activation, _INFERRED, Parameter, Placeholder, Block, init_default_or_glorot_uniform +from cntk.layers import Recurrence, Convolution +from cntk.initializer import uniform, glorot_uniform +from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input +from cntk.utils.debughelpers import _name_node, _node_name, _node_description, _log_node + + +######################## +# variables and stuff # +######################## + +data_dir = "./data" +model_dir = "./models" + +# model dimensions +#vocab_dim = 40000 +#embed_dim = 200 +#hidden_dim = 256 +#src_max_len = 120 +#ctx_max_len = 1989 + +# stabilizer +stabilize = Stabilizer() + +def text_convolution(win_size, in_dim, out_dim): + #activation = _resolve_activation(activation) + output_channels_shape = _as_tuple(out_dim) + output_rank = len(output_channels_shape) + filter_shape = (win_size, in_dim) + filter_rank = len(filter_shape) + kernel_shape = _INFERRED + filter_shape # kernel := filter plus reductionDims + + # parameters bound to this Function + init_kernel = glorot_uniform(filter_rank=filter_rank, output_rank=1) + #init_kernel = _initializer_for(init, Record(filter_rank=filter_rank, output_rank=-1)) + # BUGBUG: It is very confusing that output_rank is negative, esp. since that means count from the start. Solution: add a flag + W = Parameter(output_channels_shape + kernel_shape, init=init_kernel, name='W') # (K, C, H, W) aka [ W x H x C x K ] + #w = np.reshape(np.array([[[[2, -1, 0, -1, 2],[1,1,2,-1,-1],[1,2,0,2,1]]]], dtype = np.float32), (1, 1, 3, 5)) + #W = constant(value=w) + #b = Parameter(output_channels_shape + (1,) * len(filter_shape), init=init_bias, name='b') if bias else None # (K, 1, 1) aka [ 1 x 1 x K ] + + # expression + x = Placeholder(name='convolution_arg') + # TODO: update the parameter order of convolution() to match the optional ones as in here? (options order matches Keras) + strides = (1, 1, in_dim) + + apply_x = convolution (W, x, + strides = _as_tuple(strides), + sharing = _as_tuple(True), + auto_padding = _as_tuple(False), + lower_pad = (0, win_size/2, 0), + upper_pad = (0, (win_size-1)/2, 0) + ) +# # TODO: can we rename auto_padding to pad? + #if bias: + # apply_x = apply_x + b + apply_x = apply_x >> sigmoid + return Block(apply_x, 'Convolution', Record(W=W)) + +def create_reader(path, randomize, size=INFINITELY_REPEAT): + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + answer = StreamDef(field='A', shape=vocab_dim, is_sparse=True) + )), randomize=randomize, epoch_size = size) + + ######################## +# define the model # +######################## + +def gru_cell(shape, init=init_default_or_glorot_uniform, name=''): # (x, (h,c)) + shape = _as_tuple(shape) + + if len(shape) != 1 : + raise ValueError("gru_cell: shape must be vectors (rank-1 tensors)") + + # determine stacking dimensions + cell_shape_stacked = shape * 2 # patched dims with stack_axis duplicated 4 times + + # parameters + Wz = Parameter(cell_shape_stacked, init = init, name='Wz') + Wr = Parameter(cell_shape_stacked, init = init, name='Wr') + Wh = Parameter(cell_shape_stacked, init = init, name='Wh') + Uz = Parameter( _INFERRED + shape, init = init, name = 'Uz') + Ur = Parameter( _INFERRED + shape, init = init, name = 'Ur') + Uh = Parameter( _INFERRED + shape, init = init, name = 'Uh') + + def create_s_placeholder(): + # we pass the known dimensions here, which makes dimension inference easier + return Placeholder(shape=shape, name='S') # (h, c) + + # parameters to model function + x = Placeholder(name='gru_block_arg') + prev_status = create_s_placeholder() + + # formula of model function + Sn_1 = prev_status + + z = sigmoid(times(x, Uz, name='x*Uz') + times(Sn_1, Wz, name='Sprev*Wz'), name='z') + r = sigmoid(times(x, Ur, name='x*Ur') + times(Sn_1, Wr, name='Sprev*Wr'), name='r') + h = tanh(times(x, Uh, name='x*Uh') + times(element_times(Sn_1, r, name='Sprev*r'), Wh), name='h') + s = plus(element_times((1-z), h, name='(1-z)*h'), element_times(z, Sn_1, name='z*SPrev'), name=name) + apply_x_s = combine([s]) + apply_x_s.create_placeholder = create_s_placeholder + return apply_x_s + +def bidirectionalLSTM(hidden_dim, x, splice_outputs=True): + fwd = Recurrence(LSTM(hidden_dim), go_backwards=False) (stabilize(x)) + bwd = Recurrence(LSTM(hidden_dim), go_backwards=True ) (stabilize(x)) + if splice_outputs: + # splice the outputs together + hc = splice((fwd, bwd)) + return hc + else: + # return both (in cases where we want the 'final' hidden status) + return (fwd, bwd) + +def bidirectional_gru(hidden_dim, x, splice_outputs=True, name=''): + fwd = Recurrence(gru_cell(hidden_dim), go_backwards=False) (stabilize(x)) + bwd = Recurrence(gru_cell(hidden_dim), go_backwards=True) (stabilize(x)) + if splice_outputs: + # splice the outputs together + hc = splice((fwd, bwd), name=name) + return hc + else: + # return both (in cases where we want the 'final' hidden status) + return (fwd, bwd) + +def broadcast_as(op, seq_op, name=''): + x=placeholder_variable(shape=op.shape, name='src_op') + s=placeholder_variable(shape=seq_op.shape, name='tgt_seq') + pd = sequence.scatter(x, sequence.is_first(s, name='isf_1'), name='sct') + pout = placeholder_variable(op.shape, dynamic_axes=seq_op.dynamic_axes, name='pout') + out = element_select(sequence.is_first(s, name='isf_2'), pd, past_value(pout, name='ptv'), name='br_sel') + rlt = out.replace_placeholders({pout:sanitize_input(out), x:sanitize_input(op), s:sanitize_input(seq_op)}) + return combine([rlt], name=name) + +def cosine_similarity(src, tgt, name=''): + src_br = broadcast_as(src, tgt, name='cos_br') + sim = cosine_distance(src_br, tgt, name=name) + return sim + +def project_cosine_sim(status, memory, dim, init = init_default_or_glorot_uniform, name=''): + cell_shape = (dim, dim) + Wi = Parameter(cell_shape, init = init, name='Wi') + Wm = Parameter(cell_shape, init = init, name='Wm') + weighted_status = times(status, Wi, name = 'project_status') + weighted_memory = times(memory, Wm, name = 'project_memory') + return cosine_similarity(weighted_status, weighted_memory, name=name) + +def termination_gate(status, dim, init = init_default_or_glorot_uniform, name=''): + Wt = Parameter((dim, 1), init = init, name='Wt') + return sigmoid(times(status, Wt), name=name) + +def attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids,hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): + status = Placeholder(name='status', shape=hidden_dim) + context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim, name='context_attention') + query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim, name='query_attetion') + context_attention = sequence.reduce_sum(element_times(context_attention_weight, context_memory), name='C-Att') + query_attention = sequence.reduce_sum(element_times(query_attention_weight, query_memory), name='Q-Att') + attention = splice((query_attention, context_attention), name='att-sp') + gru = gru_cell((hidden_dim, ), name='status') + new_status = gru(attention, status).output + termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') + ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim, name='ans_attention') + answers = times(ans_attention, candidate_ids, name='answers') + return combine([answers, termination_prob, new_status], name='ReinforcementAttention') + +def attention_rlunit2(hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): + status = Placeholder(name='status') + context_memory = Placeholder(name='context_memory') + query_memory = Placeholder(name='query_memory') + candidate_memory = Placeholder(name='candidate_memory') + candidate_ids = Placeholder(name='candidate_ids') + context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim) + query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim) + context_attention = reduce_sum(element_times(context_attention_weight, context_memory), axis = 0) + query_attention = reduce_sum(element_times(query_attention_weight, query_memory), axis = 0) + attention = splice((query_attention, context_attention)) + gru = gru_cell((hidden_dim, ), name='status') + new_status = gru(attention, status).output + termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') + ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim) + answers = times(ans_attention, candidate_ids, name='ans2') + return combine([answers, termination_prob, new_status], name='ReinforcementAttention') + +def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glorot_uniform): + # Query and Doc/Context/Paragraph inputs to the model + batch_axis = Axis.default_batch_axis() + query_seq_axis = Axis('sourceAxis') + context_seq_axis = Axis('contextAxis') + query_dynamic_axes = [batch_axis, query_seq_axis] + query_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=query_dynamic_axes, name='query') + context_dynamic_axes = [batch_axis, context_seq_axis] + context_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') + candidate_dynamic_axes = [batch_axis, context_seq_axis] + candidate_indicates = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') + candidate_filter = greater(candidate_indicates, 0) + + # Query sequences + query_sequence = query_raw + # Doc/Context sequences + context_sequence = context_raw + # embedding + embed_dim = hidden_dim + embedding = parameter(shape=(vocab_dim, embed_dim), init=uniform(1)) + + query_embedding = times(query_sequence , embedding) + context_embedding = times(context_sequence, embedding) + + # get source and context representations + context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis + #candidate_memory = sequence.gather(context_memory, candidate_filter, name='Candidate_Mem') + candidate_memory = context_memory + candidate_ids = candidate_indicates + + qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis + query_memory = splice((qfwd, qbwd), name='Query_SP') + # get the source (aka 'query') representation + status = splice((sequence.last(qfwd), sequence.first(qbwd)), name='Init_Status') # get last fwd status and first bwd status + attention_rlu = attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids, hidden_dim*2, vocab_dim, init) + status_controls = [] + arlus = [None] * max_rl_iter + answers = None + probs = None + for i in range(0, max_rl_iter): + #arlus[i] = attention_rlu(status, context_memory, query_memory, candidate_memory, candidate_ids) + arlus[i] = attention_rlu(status) + status = arlus[i].outputs[2] + status_controls += list(arlus[i].outputs[0:2]) + if answers == None: + answers = element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) + #answers = element_times(sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0]), arlus[i].outputs[0]) + probs = arlus[i].outputs[1] + else: + answers += element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) + #answers += element_times(sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0]), arlus[i].outputs[0]) + probs += arlus[i].outputs[1] + final_answers = element_divide(answers, sequence.broadcast_as(probs, answers), name='final_answers') + return combine(status_controls+[final_answers], name='ReasoNet') + +def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + entities = StreamDef(field='E', shape=1, is_sparse=False), + label = StreamDef(field='L', shape=1, is_sparse=False) + )), randomize=randomize, epoch_size = size) diff --git a/Examples/ReasoNet/test.idx b/Examples/ReasoNet/test.idx new file mode 100644 index 000000000000..ac9dc92a9a34 --- /dev/null +++ b/Examples/ReasoNet/test.idx @@ -0,0 +1,5 @@ +0 |Q 8:1 |C 4:1 |E 0 |L 0 +0 |Q 8:1 |C 3:1 |E 1 |L 0 +0 |Q 9:1 |C 5:1 |E 0 |L 0 +0 |Q 1:1 |C 6:1 |E 0 |L 0 +0 |C 3:1 |E 1 |L 1 diff --git a/Examples/ReasoNet/test.py b/Examples/ReasoNet/test.py new file mode 100644 index 000000000000..33bbc562a4d4 --- /dev/null +++ b/Examples/ReasoNet/test.py @@ -0,0 +1,159 @@ +import sys +import os +import numpy as np +from ReasoNet.model import text_convolution +from ReasoNet.model import gru_cell +from cntk.blocks import Placeholder, Constant +from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ + parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, cosine_distance, times_transpose +from cntk import Axis + +def testTextConvolution(): + text = np.reshape(np.arange(25.0, dtype=np.float32), (1, 5,5)) + x = input_variable(shape=(1, 5, 5,)) + c = text_convolution(3, 5, 5)(x) + v = c.eval([text])[0] + print(v) + +def testSplice(): + a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) + b = np.reshape(np.arange(0, -25, -1, dtype=np.float32), (1,5,5)) + va = constant(value=a) + vb = constant(value=b) + vc = splice((va, vb), 2) + print(vc.shape) + print(vc.eval()) + +def cossim(a,b): + src = constant(a) + tgt = constant(b) + val = cosine_distance(src, tgt).eval() + return val + +def testCosinDistance(): + a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) + b = np.reshape(np.arange(0, 5, dtype=np.float32), (1,5)) + + src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) + tgt = input_variable(shape=(5)) + tgt_br = sequence.broadcast_as(tgt, src) + cos_seq = cosine_distance(src, tgt_br) + val = cos_seq.eval({src:[a], tgt:[b]}) + print("Cosine similarity\r\n{0}\r\n #\r\n{1}".format(a,b)) + print("==>") + print(val) + print("==================") + print("Expected: ") + for i in range(0, 5): + print("{0}:{1}".format(i, cossim(a[i], b[0]))) + +def dotproduct(a,b): + src = constant(a) + tgt = constant(b) + val = reduce_sum(element_times(src, tgt)) + return val + +def testReduceSum(): + a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) + b = np.reshape(np.arange(0, 5, dtype=np.float32), (1,5)) + + src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) + tgt = input_variable(shape=(5)) + tgt_br = sequence.broadcast_as(tgt, src) + reduceSum = reduce_sum(element_times(src, tgt_br), axis=0) + val = reduceSum.eval({src:[a], tgt:[b]}) + print("Reduce_sum\r\n{0}\r\n #\r\n{1}".format(a,b)) + print("==>") + print(val) + print("==================") + print("Expected: ") + for i in range(0, 5): + print("{0}:{1}".format(i, dotproduct(a[i], b[0]).eval())) + + +def testElementTimes(): + a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) + b = np.reshape(np.arange(0, 5, dtype=np.float32), (5)) + +# src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) +# tgt = input_variable(shape=(5)) +# tgt_br = sequence.broadcast_as(tgt, src) +# reduceSum = reduce_sum(element_times(src, tgt_br), axis=0) + val = element_times(b.reshape(5,1),a).eval() + print("ElementTimes\r\n{0}\r\n #\r\n{1}".format(a,b)) + print("==>") + print(val) + print("==================") + print("Expected: ") + for i in range(0, 5): + print("{0}:{1}".format(i, element_times(a[i], b[i]).eval())) + +def testElementTimes2(): + a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) + b = np.reshape(np.arange(0, 5, dtype=np.float32), (1, 5)) + +# src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) +# tgt = input_variable(shape=(5)) +# tgt_br = sequence.broadcast_as(tgt, src) +# reduceSum = reduce_sum(element_times(src, tgt_br), axis=0) + val = reduce_sum(element_times(b,a).eval(), axis=1).eval() + print("ElementTimes\r\n{0}\r\n #\r\n{1}".format(a,b)) + print("==>") + print(val) + print("==================") + print("Expected: ") + for i in range(0, 5): + print("{0}:{1}".format(i, dotproduct(a[i], b[0]).eval())) + +def testTimesTranspose(): + a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) + b = np.reshape(np.arange(0, 5, dtype=np.float32), (1, 5)) + +# src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) +# tgt = input_variable(shape=(5)) +# tgt_br = sequence.broadcast_as(tgt, src) +# reduceSum = reduce_sum(element_times(src, tgt_br), axis=0) + val = times_transpose(a,b).eval() + print("ElementTimes\r\n{0}\r\n #\r\n{1}".format(a,b)) + print("==>") + print(val) + print("==================") + print("Expected: ") + for i in range(0, 5): + print("{0}:{1}".format(i, dotproduct(a[i], b[0]).eval())) + +def testGRU(): + a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) + b = np.reshape(np.arange(0, 5, dtype=np.float32), (1,5)) + src = input_variable(shape=(5, )) + #src = input_variable(shape=(1, 5, ), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) + tgt = constant(b) + gru = gru_cell(5) + o_0 = gru(src, tgt) + sgru = gru(src, o_0.output).output[0] + print(sgru.eval(a)) + + +def reduce_times_sum(first, second): + # define a recursive expression for \sum_{i=1}^t (first_i * second_i) + running_sum_ph = placeholder_variable(shape=first.shape) + print("Second: {0}".format(second.shape)) + print("Fist:{0}".format(first.shape)) + t = times(second, first) + print("Times: {0}".format(t.output.shape)) + return t + #running_sum = plus(reshape(times(second, first), shape=(5)), past_value(running_sum_ph)) + + #print("Plus: {0}".format(running_sum.output.shape)) + #running_sum.replace_placeholders({running_sum_ph : running_sum.output}) + #return sequence.last(running_sum) + +def testReduceTimesSum(): + a = np.reshape(np.float32([[0,1,0,0,0],[1,0,0,0,0],[1,0,0,0,0], [0,0,0,1,0],[0,0,0,0,1]]), (5,5)) + b = np.reshape(np.float32([0.1, 0.2, 0.2, 0.1, 0.4]), (1,5)) + src = input_variable(shape=(5, 5,)) +# src = input_variable(shape=(1, 5, ), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) + tgt = constant(b) + print(reduce_times_sum(src,tgt).eval([a])) + + diff --git a/Examples/ReasoNet/tests/testReasoNetModel.py b/Examples/ReasoNet/tests/testReasoNetModel.py new file mode 100644 index 000000000000..c8e1fff71ea2 --- /dev/null +++ b/Examples/ReasoNet/tests/testReasoNetModel.py @@ -0,0 +1,64 @@ +import sys +import os +import numpy as np +from ReasoNet.model import create_model, create_reader, gru_cell +import ReasoNet.asr as asr +from cntk.blocks import Placeholder, Constant,initial_state_default_or_None, _is_given, _get_current_default_options +from cntk.ops import input_variable, past_value, future_value +from cntk.io import MinibatchSource +from cntk import Trainer, Axis, device, combine +from cntk.layers import Recurrence, Convolution +from cntk.utils import _as_tuple + +def testModel(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + data_bind = {} + for arg in model.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + batch = reader.next_minibatch(1, data_bind) + var = model.eval(batch) + for o in model.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + +def testASR(data): + reader = asr.create_reader(data, 10, False) + model = asr.create_model(10, 5, 1) + data_bind = {} + for arg in model.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + batch = reader.next_minibatch(1, data_bind) + var = model.eval(batch) + for o in model.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + + +def testGRU(): + g = gru_cell(5) + x = np.reshape(np.arange(0,25, dtype=np.float32), (1,5,5)) + a = input_variable(shape=(5,), dynamic_axes=[Axis.default_batch_axis(), Axis('Seq')]) + y = np.float32([1,2,0.1,0.2,1]) + s = Constant(y) + q = g(a,s).eval({a:x}) + print(q) + r = Recurrence(gru_cell(5)) + rt = r(a).eval({a:x}) + print(rt) + +#testGRU() +#testASR("test.idx") +testModel("test.idx") diff --git a/Examples/ReasoNet/tests/testTextConvolution.py b/Examples/ReasoNet/tests/testTextConvolution.py new file mode 100644 index 000000000000..db62f28a17e3 --- /dev/null +++ b/Examples/ReasoNet/tests/testTextConvolution.py @@ -0,0 +1,13 @@ +import sys +import os +import numpy as np +from ReasoNet.model import text_convolution +from cntk.blocks import Placeholder, Constant +from cntk.ops import input_variable + +def testTextConvolution(): + text = np.reshape(np.arange(25.0, dtype=np.float32), (1, 5,5)) + x = input_variable(shape=(1, 5, 5,)) + c = text_convolution(3, 5, 5)(x) + v = c.eval([text])[0] + print(v) diff --git a/Examples/ReasoNet/wordvocab.py b/Examples/ReasoNet/wordvocab.py new file mode 100644 index 000000000000..d1ac34eb3b73 --- /dev/null +++ b/Examples/ReasoNet/wordvocab.py @@ -0,0 +1,200 @@ +import sys +import os + +class WordFreq: + def __init__(self, word, id, freq): + self.word = word + self.id = id + self.freq = freq + +class Vocabulary: + """Build word vocabulary with frequency""" + def __init__(self, name): + self.name = name + self.size = 0 + self.__dict = {} + self.__has_index = False + + def push(self, word): + if word in self.__dict: + self.__dict[word].freq += 1 + else: + self.__dict[word] = WordFreq(word, len(self.__dict), 1) + + def build_index(self, max_size): + items = sorted(self.__dict.values(), key=lambda it : it.freq, reverse=True) + if len(items)>max_size: + del items[max_size:] + self.size=len(items) + self.__dict.clear() + for it in items: + it.id = len(self.__dict) + self.__dict[it.word] = it + self.__has_index = True + + def save(self, dst): + if not self.__has_index: + self.build_index(sys.maxsize) + if self.name != None: + dst.write("{0}\t{1}\n".format(self.name, self.size)) + for it in sorted(self.__dict.values(), key=lambda it:it.id): + dst.write("{0}\t{1}\t{2}\n".format(it.word, it.id, it.freq)) + + def load(self, src): + line = src.readline() + if line == "": + return + head = line.split() + max_size = sys.maxsize + if len(head) == 2: + self.name = head[0] + max_size = int(head[1]) + cnt = 0 + while cnt < max_size: + line = src.readline() + if line == "": + break + items = line.split() + self.__dict[items[0]] = WordFreq(items[0], int(items[1]), int(items[2])) + cnt += 1 + self.size = len(self.__dict) + self.__has_index = True + + def __getitem__(self, key): + if key in self.__dict: + return self.__dict[key] + else: + return None + + @staticmethod + def load_bingvocab(vocab_src): + """ + Load bing vocabulary from file. + + Args: + vocab_src (`str`): the file stored with the vocabulary data + Returns: + :class:`Vocabulary`: Vocabulary of the entities + :class:`Vocabulary`: Vocabulary of the words + """ + word_vocab = Vocabulary("WordVocab") + entity_vocab = Vocabulary("EntityVocab") + with open(vocab_src, 'r', encoding='utf-8') as src: + entity_vocab.load(src) + word_vocab.load(src) + return entity_vocab, word_vocab + + @staticmethod + def build_bingvocab(input_src, vocab_dst, max_size=50000): + """ + Build bing vocabulary from raw bing corpus file. + + Args: + input_src (`str`): the path of the corpus file + vocab_dst (`str`): the path of the vocabulary file to save the built vocabulary + max_size (`int`): the maxium size of the word vocabulary + Returns: + :class:`Vocabulary`: Vocabulary of the entities + :class:`Vocabulary`: Vocabulary of the words + """ + word_vocab = Vocabulary("WordVocab") + entity_vocab = Vocabulary("EntityVocab") + linenum = 0 + with open(input_src, 'r', encoding='utf-8') as src: + for line in src.readlines(): + ans, query_words, context_words = Vocabulary.parse_bing_corpus_line(line) + for q in query_words: + if q.startswith('@'): + entity_vocab.push(q) + else: + word_vocab.push(q) + for q in context_words: + if q.startswith('@'): + entity_vocab.push(q) + else: + word_vocab.push(q) + linenum += 1 + if linenum%1000==0: + print("{0} lines parsed.".format(linenum)) + entity_vocab.build_index(max_size) + word_vocab.build_index(max_size) + with open(vocab_dst, 'w', encoding='utf-8') as dst: + entity_vocab.save(dst) + word_vocab.save(dst) + return entity_vocab, word_vocab + + @staticmethod + def parse_bing_corpus_line(line): + """ + Parse bing corpus line to answer, query and context. + + Args: + line (`str`): A line of text of bing corpus + Returns: + :`str`: Answer word + :`str[]`: Array of query words + :`str[]`: Array of context/passage words + + """ + data = line.split('\t') + query = data[0] + answer = data[1] + context = data[2] + query_words = query.split() + context_words = context.split() + return answer, query_words, context_words + + @staticmethod + def build_bing_corpus_index(entities, words, corpus, index): + """ + Build featurized corpus and stored in index file in CNTKTextFormat. + + Args: + entities (class:`Vocabulary`): The entities vocabulary + words (class:`Vocabulary`): The words vocabulary + corpus (`str`): The file path of the raw corpus + index (`str`): The file path to store the featurized corpus index + """ + seq_id = 0 + with open(corpus, 'r', encoding = 'utf-8') as corp: + with open(index, 'w', encoding = 'utf-8') as index: + for line in corp.readlines(): + ans, query_words, context_words = Vocabulary.parse_bing_corpus_line(line) + ans_item = entities[ans] + query_ids = [] + context_ids = [] + is_entity = [] + labels = [] + pos = 0 + for q in context_words: + if q.startswith('@'): + item = entities[q] + context_ids += [ item.id + 1 ] + is_entity += [1] + labels += [1 if ans_item.id==item.id else 0] + else: + item = words[q] + context_ids += [ (item.id + 1) if item != None else 0 ] + is_entity += [0] + labels += [0] + pos += 1 + for q in query_words: + if q.startswith('@'): + item = entities[q] + query_ids += [ item.id + 1 ] + else: + item = words[q] + query_ids += [ (item.id + 1) if item != None else 0 ] + #Write featurized ids + index.write("{0}".format(seq_id)) + for i in range(max(len(context_ids), len(query_ids))): + if i < len(query_ids): + index.write(" |Q {0}".format(query_ids[i])) + if i < len(context_ids): + index.write(" |C {0}".format(context_ids[i])) + index.write(" |E {0}".format(is_entity[i])) + index.write(" |L {0}".format(labels[i])) + index.write("\n") + seq_id += 1 + if seq_id%1000 == 0: + print("{0} lines parsed.".format(seq_id)) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index e2971b8ce287..1b8274362059 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -364,7 +364,7 @@ namespace CNTK if (i != 0) wStrStream << L" x "; - wStrStream << m_shapeDims[i]; + wStrStream << ((int)m_shapeDims[i]); } wStrStream << L"]"; diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.cpp b/Source/CNTKv2LibraryDll/CompositeFunction.cpp index a97db537f569..fff2a2b968bb 100644 --- a/Source/CNTKv2LibraryDll/CompositeFunction.cpp +++ b/Source/CNTKv2LibraryDll/CompositeFunction.cpp @@ -339,6 +339,34 @@ namespace CNTK // when the new CNTK v2 model serialization format is ready. /*static*/ const std::wstring CompositeFunction::InternalDefaultDynamicAxisName = L"*"; /*static*/ const std::wstring CompositeFunction::InternalNoSequenceAxisName = L"__noSequenceAxis"; + + void DumpInputsTree(const Variable &var, const int &level) + { + auto owner = var.Owner(); + if(level>20) + return; + + if(owner == nullptr) + return; + auto p_inputs = owner->Inputs(); + + if(p_inputs.size() == 0) + return; + + fprintf(stderr, ">>>\n"); + for(auto iter = p_inputs.begin(); iter != p_inputs.end(); iter++) + { + fprintf(stderr, "%d|%S<=Input %S with shape %S!\n", level, ParanthesizedName(owner->Name()).c_str(), ParanthesizedName(iter->Name()).c_str(), iter->Shape().AsString().c_str()); + } + fprintf(stderr, "===\n"); + for(auto iter = p_inputs.begin(); iter != p_inputs.end(); iter++) + { + if(iter->Shape().IsUnknown()) + { + DumpInputsTree(*iter, level+1); + } + } + } // Recursively create a sub-network of ComputationNode instances corresponding to the graph of Functions // underlying the specified 'variable' and return the ComputationNode instance that corresponds to the @@ -362,7 +390,13 @@ namespace CNTK InvalidArgument("Variable%S with unknown DataType detected when compiling the Function graph!", ParanthesizedName(variable.Name()).c_str()); if (variable.Shape().IsUnknown()) - InvalidArgument("Variable%S with unknown shape detected when compiling the Function graph!", ParanthesizedName(variable.Name()).c_str()); + { + + fprintf(stderr, "Variable%S with unknown shape detected when compiling the Function graph, owner: %S!\n", ParanthesizedName(variable.Name()).c_str(), ParanthesizedName(variable.Owner()->Name()).c_str()); + DumpInputsTree(variable, 0); + fprintf(stderr, "Quit\n"); + InvalidArgument("Variable%S with unknown shape detected when compiling the Function graph!", ParanthesizedName(variable.Name()).c_str()); + } if (variable.Shape().HasInferredDimension()) InvalidArgument("Variable%S with InferredDimension for at least one axis in its shape, detected when compiling the Function graph!", ParanthesizedName(variable.Name()).c_str()); diff --git a/bindings/python/cntk/layers.py b/bindings/python/cntk/layers.py index 6bf2882fbe6f..0eefb3804f33 100644 --- a/bindings/python/cntk/layers.py +++ b/bindings/python/cntk/layers.py @@ -234,7 +234,11 @@ def previous_hook(state): f_x_h_c = over(x, prev_state) # apply the recurrent over # this returns a Function (x, (h_prev, c_prev)) -> (h, c) h_c = f_x_h_c.outputs - replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) } + if type(state_forward) is tuple and len(state_forward) > 1: + replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) } + else: + replacements = {(state_forward,)[0] : h_c[0] } + f_x_h_c.replace_placeholders(replacements) # resolves state_forward := h_c h = f_x_h_c.outputs[0] # 'h' is a Variable (the output of a Function that computed it) if _trace_layers: diff --git a/bindings/python/setup.py b/bindings/python/setup.py index e1187cfb6bc7..d771f124a249 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -42,7 +42,7 @@ CNTK_LIB_PATH = os.path.join(CNTK_PATH, "x64", "Release") else: CNTK_LIB_PATH = os.path.join( - CNTK_PATH, "build", "gpu", "release", "lib") + CNTK_PATH, "build", "gpu", "debug", "lib") print("Using CNTK sources at '%s'" % os.path.abspath(CNTK_SOURCE_PATH)) print("Using CNTK libs at '%s'" % os.path.abspath(CNTK_LIB_PATH)) From 2f94c5430bfb9a235466c29c3dd095f9acf56371 Mon Sep 17 00:00:00 2001 From: PengchengHe Date: Wed, 4 Jan 2017 11:55:28 +0000 Subject: [PATCH 040/120] Add loss/pred for ReasoNet --- Examples/ReasoNet/model.py | 79 ++++++++++++--- Examples/ReasoNet/test.idx | 7 +- Examples/ReasoNet/testReasoNetModel.py | 119 +++++++++++++++++++++++ Examples/ReasoNet/testTextConvolution.py | 13 +++ 4 files changed, 202 insertions(+), 16 deletions(-) create mode 100644 Examples/ReasoNet/testReasoNetModel.py create mode 100644 Examples/ReasoNet/testTextConvolution.py diff --git a/Examples/ReasoNet/model.py b/Examples/ReasoNet/model.py index b0eb2eee0062..77026b524073 100644 --- a/Examples/ReasoNet/model.py +++ b/Examples/ReasoNet/model.py @@ -10,10 +10,11 @@ from cntk.learner import momentum_sgd, momentum_as_time_constant_schedule, learning_rate_schedule, UnitType from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, tanh, times_transpose, greater, cosine_distance, element_divide, element_select +import cntk.ops as ops from cntk.blocks import LSTM, Stabilizer, _get_current_default_options, _is_given, _initializer_for, _resolve_activation, _INFERRED, Parameter, Placeholder, Block, init_default_or_glorot_uniform from cntk.layers import Recurrence, Convolution from cntk.initializer import uniform, glorot_uniform -from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input +from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input, value_to_seq from cntk.utils.debughelpers import _name_node, _node_name, _node_description, _log_node @@ -196,6 +197,23 @@ def attention_rlunit2(hidden_dim, vocab_dim, init = init_default_or_glorot_unifo answers = times(ans_attention, candidate_ids, name='ans2') return combine([answers, termination_prob, new_status], name='ReinforcementAttention') +def seq_cross_entropy(pred, label, gama=10, name=''): + pred_exp = ops.exp(pred, name='pred_exp') + sum_exp = sequence.reduce_sum(pred_exp, name='sum_exp') + pred_sum = element_divide(pred_exp, sequence.broadcast_as(sum_exp, pred), name='exp_divid') + log_pred_sum = ops.log(pred_sum, name='log_pred') + label_pred = times(label, log_pred_sum, name = 'label_softmax') + entroy = ops.negate(sequence.reduce_sum(label_pred, name='sum_log'), name=name) + #loss = ops.negate(sequence.reduce_sum(times(label, ops.log(pred_exp/(sequence.broadcast_as(sum_exp, pred))))), name = name) + return entroy + +def mask_cross_entropy(pred, label, mask, gama=10, name=''): + pred_exp = element_select(mask, ops.exp(gama*pred), 0) + label_msk = element_select(label, 1, 0) + sum_exp = ops.reduce_sum(pred_exp) + soft_max = ops.element_select(mask, ops.negate(ops.element_times(label_msk, ops.log(pred_exp/sum_exp))), 0) + return ops.reduce_sum(soft_max, name=name) + def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glorot_uniform): # Query and Doc/Context/Paragraph inputs to the model batch_axis = Axis.default_batch_axis() @@ -207,7 +225,6 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor context_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') candidate_dynamic_axes = [batch_axis, context_seq_axis] candidate_indicates = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') - candidate_filter = greater(candidate_indicates, 0) # Query sequences query_sequence = query_raw @@ -221,10 +238,12 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor context_embedding = times(context_sequence, embedding) # get source and context representations - context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis - #candidate_memory = sequence.gather(context_memory, candidate_filter, name='Candidate_Mem') - candidate_memory = context_memory - candidate_ids = candidate_indicates + context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis + candidate_filter = greater(candidate_indicates, 0) + candidate_sc = sequence.gather(candidate_filter, candidate_filter) + candidate_memory = sequence.scatter(sequence.gather(context_memory, candidate_filter, name='Candidate_Mem'), candidate_sc) + candidate_ids = sequence.scatter(sequence.gather(candidate_indicates, candidate_filter, name = 'Candidate_Ids'), candidate_sc) + entity_raw_ids = sequence.scatter(sequence.gather(reshape(context_raw, vocab_dim), candidate_filter), candidate_sc) qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis query_memory = splice((qfwd, qbwd), name='Query_SP') @@ -236,25 +255,55 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor answers = None probs = None for i in range(0, max_rl_iter): - #arlus[i] = attention_rlu(status, context_memory, query_memory, candidate_memory, candidate_ids) arlus[i] = attention_rlu(status) status = arlus[i].outputs[2] status_controls += list(arlus[i].outputs[0:2]) if answers == None: answers = element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) - #answers = element_times(sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0]), arlus[i].outputs[0]) probs = arlus[i].outputs[1] else: answers += element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) - #answers += element_times(sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0]), arlus[i].outputs[0]) probs += arlus[i].outputs[1] final_answers = element_divide(answers, sequence.broadcast_as(probs, answers), name='final_answers') - return combine(status_controls+[final_answers], name='ReasoNet') + result = combine(status_controls+[final_answers], name='ReasoNet') + return Block(result, 'ReasoNet', Record(vocab_dim=vocab_dim, hidden_dim=hidden_dim, max_ite =max_rl_iter, context=context_raw, + query=query_raw, entities=candidate_indicates, entity_masks=candidate_filter, + entity_seqs=candidate_sc, entity_ids=entity_raw_ids)) + +def pred(model): + context = model.context + entities = model.entities + wordvocab_dim = model.vocab_dim + candidate_filter = model.entity_masks + candidate_sc = model.entity_seqs + answers = sequence.scatter(model.outputs[-1], candidate_sc, name='answers_prob') + entity_ids = model.entity_ids + item_preds = sequence.reduce_sum(times(reshape(answers, (1,)), entity_ids), name = 'item_preds') + mask = sequence.reduce_sum(entity_ids, name='mask') + probs = ops.element_select(mask, ops.exp(item_preds), 0, name='item_prob') + return combine([mask, probs]) + +def loss(model): + context = model.context + entities = model.entities + wordvocab_dim = model.vocab_dim + labels_raw = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context.dynamic_axes, name='labels') + candidate_filter = model.entity_masks + candidate_sc = model.entity_seqs + answers = sequence.scatter(model.outputs[-1], candidate_sc, name='answers_prob') + entity_ids = model.entity_ids + item_preds = sequence.reduce_sum(times(reshape(answers, (1,)), entity_ids), name = 'item_preds') + labels = sequence.scatter(sequence.gather(labels_raw, candidate_filter, name='EntityLabels'), candidate_sc, name='seq_labels') + #cross_entroy = seq_cross_entroy(reshape(answers, (1,)), labels, name='CrossEntropyLoss') + item_labels = sequence.reduce_sum(times(reshape(labels, (1,)), entity_ids), name='item_labels') + mask = sequence.reduce_sum(entity_ids) + cross_entroy = mask_cross_entropy(item_preds, item_labels, mask, name='CrossEntropyLoss') + return combine([cross_entroy, answers, labels, item_preds]) def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - entities = StreamDef(field='E', shape=1, is_sparse=False), - label = StreamDef(field='L', shape=1, is_sparse=False) + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + entities = StreamDef(field='E', shape=1, is_sparse=False), + label = StreamDef(field='L', shape=1, is_sparse=False) )), randomize=randomize, epoch_size = size) diff --git a/Examples/ReasoNet/test.idx b/Examples/ReasoNet/test.idx index ac9dc92a9a34..0316597a4c17 100644 --- a/Examples/ReasoNet/test.idx +++ b/Examples/ReasoNet/test.idx @@ -1,5 +1,10 @@ 0 |Q 8:1 |C 4:1 |E 0 |L 0 -0 |Q 8:1 |C 3:1 |E 1 |L 0 +0 |Q 8:1 |C 2:1 |E 1 |L 0 0 |Q 9:1 |C 5:1 |E 0 |L 0 0 |Q 1:1 |C 6:1 |E 0 |L 0 0 |C 3:1 |E 1 |L 1 +1 |Q 2:1 |C 4:1 |E 0 |L 0 +1 |Q 1:1 |C 9:1 |E 1 |L 1 +1 |Q 7:1 |C 1:1 |E 0 |L 0 +1 |Q 1:1 |C 4:1 |E 0 |L 0 +1 |C 3:1 |E 1 |L 0 diff --git a/Examples/ReasoNet/testReasoNetModel.py b/Examples/ReasoNet/testReasoNetModel.py new file mode 100644 index 000000000000..3513f17a6fa9 --- /dev/null +++ b/Examples/ReasoNet/testReasoNetModel.py @@ -0,0 +1,119 @@ +import sys +import os +import numpy as np +from ReasoNet.model import create_model, create_reader, gru_cell +import ReasoNet.model as rsn +import ReasoNet.asr as asr +from cntk.blocks import Placeholder, Constant,initial_state_default_or_None, _is_given, _get_current_default_options +from cntk.ops import input_variable, past_value, future_value +from cntk.io import MinibatchSource +from cntk import Trainer, Axis, device, combine +from cntk.layers import Recurrence, Convolution +from cntk.utils import _as_tuple + +def testModel(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + data_bind = {} + for arg in model.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + batch = reader.next_minibatch(2, data_bind) + var = model.eval(batch) + for o in model.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + +def testReasoNetLoss(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + loss = rsn.loss(model) + data_bind = {} + for arg in loss.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + if arg.name == 'labels': + data_bind[arg] = reader.streams.label + batch = reader.next_minibatch(100, data_bind) + var = loss.eval(batch) + for o in loss.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + #pred = var[loss.outputs[-1]] + #for i in pred: + # print("Prediction: {0}".format(np.argmin(np.reshape(i, 10)))) + +def testReasoNetPred(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + pred = rsn.pred(model) + data_bind = {} + for arg in pred.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + if arg.name == 'labels': + data_bind[arg] = reader.streams.label + #data_bind = {model.context:reader.streams.context, model.query:reader.streams.query, model.entities:reader.streams.entities} + batch = reader.next_minibatch(100, data_bind) + var = pred.eval(batch) + for o in pred.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + pred = var[pred.outputs[-1]] + print('-----------------------') + for i in pred: + print('') + print("Prediction: {0}\n\t=>{1}\n".format(i, np.argmax(np.reshape(i, 10)))) + +def testASR(data): + reader = asr.create_reader(data, 10, False) + model = asr.create_model(10, 5, 1) + data_bind = {} + for arg in model.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + batch = reader.next_minibatch(2, data_bind) + var = model.eval(batch) + + for o in model.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + + +def testGRU(): + g = gru_cell(5) + x = np.reshape(np.arange(0,25, dtype=np.float32), (1,5,5)) + a = input_variable(shape=(5,), dynamic_axes=[Axis.default_batch_axis(), Axis('Seq')]) + y = np.float32([1,2,0.1,0.2,1]) + s = Constant(y) + q = g(a,s).eval({a:x}) + print(q) + r = Recurrence(gru_cell(5)) + rt = r(a).eval({a:x}) + print(rt) + +#testGRU() +#testASR("test.idx") +#testModel("test.idx") +#testReasoNetLoss("test.idx") +testReasoNetPred("test.idx") diff --git a/Examples/ReasoNet/testTextConvolution.py b/Examples/ReasoNet/testTextConvolution.py new file mode 100644 index 000000000000..db62f28a17e3 --- /dev/null +++ b/Examples/ReasoNet/testTextConvolution.py @@ -0,0 +1,13 @@ +import sys +import os +import numpy as np +from ReasoNet.model import text_convolution +from cntk.blocks import Placeholder, Constant +from cntk.ops import input_variable + +def testTextConvolution(): + text = np.reshape(np.arange(25.0, dtype=np.float32), (1, 5,5)) + x = input_variable(shape=(1, 5, 5,)) + c = text_convolution(3, 5, 5)(x) + v = c.eval([text])[0] + print(v) From 732836ab001fdde86e67e29c932603205651b4d0 Mon Sep 17 00:00:00 2001 From: PengchengHe Date: Thu, 5 Jan 2017 01:54:09 +0000 Subject: [PATCH 041/120] Add training method for ReasoNet --- Examples/ReasoNet/model.py | 139 +++++++++++++------ Examples/ReasoNet/tests/testReasoNetModel.py | 67 ++++++++- 2 files changed, 164 insertions(+), 42 deletions(-) diff --git a/Examples/ReasoNet/model.py b/Examples/ReasoNet/model.py index 77026b524073..e0942299ebe0 100644 --- a/Examples/ReasoNet/model.py +++ b/Examples/ReasoNet/model.py @@ -22,9 +22,6 @@ # variables and stuff # ######################## -data_dir = "./data" -model_dir = "./models" - # model dimensions #vocab_dim = 40000 #embed_dim = 200 @@ -35,6 +32,14 @@ # stabilizer stabilize = Stabilizer() +def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + entities = StreamDef(field='E', shape=1, is_sparse=False), + label = StreamDef(field='L', shape=1, is_sparse=False) + )), randomize=randomize, epoch_size = size) + def text_convolution(win_size, in_dim, out_dim): #activation = _resolve_activation(activation) output_channels_shape = _as_tuple(out_dim) @@ -70,13 +75,6 @@ def text_convolution(win_size, in_dim, out_dim): apply_x = apply_x >> sigmoid return Block(apply_x, 'Convolution', Record(W=W)) -def create_reader(path, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - answer = StreamDef(field='A', shape=vocab_dim, is_sparse=True) - )), randomize=randomize, epoch_size = size) - ######################## # define the model # ######################## @@ -179,33 +177,15 @@ def attention_rlunit(context_memory, query_memory, candidate_memory, candidate_i answers = times(ans_attention, candidate_ids, name='answers') return combine([answers, termination_prob, new_status], name='ReinforcementAttention') -def attention_rlunit2(hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): - status = Placeholder(name='status') - context_memory = Placeholder(name='context_memory') - query_memory = Placeholder(name='query_memory') - candidate_memory = Placeholder(name='candidate_memory') - candidate_ids = Placeholder(name='candidate_ids') - context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim) - query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim) - context_attention = reduce_sum(element_times(context_attention_weight, context_memory), axis = 0) - query_attention = reduce_sum(element_times(query_attention_weight, query_memory), axis = 0) - attention = splice((query_attention, context_attention)) - gru = gru_cell((hidden_dim, ), name='status') - new_status = gru(attention, status).output - termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') - ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim) - answers = times(ans_attention, candidate_ids, name='ans2') - return combine([answers, termination_prob, new_status], name='ReinforcementAttention') - def seq_cross_entropy(pred, label, gama=10, name=''): pred_exp = ops.exp(pred, name='pred_exp') sum_exp = sequence.reduce_sum(pred_exp, name='sum_exp') pred_sum = element_divide(pred_exp, sequence.broadcast_as(sum_exp, pred), name='exp_divid') log_pred_sum = ops.log(pred_sum, name='log_pred') label_pred = times(label, log_pred_sum, name = 'label_softmax') - entroy = ops.negate(sequence.reduce_sum(label_pred, name='sum_log'), name=name) + entropy = ops.negate(sequence.reduce_sum(label_pred, name='sum_log'), name=name) #loss = ops.negate(sequence.reduce_sum(times(label, ops.log(pred_exp/(sequence.broadcast_as(sum_exp, pred))))), name = name) - return entroy + return entropy def mask_cross_entropy(pred, label, mask, gama=10, name=''): pred_exp = element_select(mask, ops.exp(gama*pred), 0) @@ -234,6 +214,8 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor embed_dim = hidden_dim embedding = parameter(shape=(vocab_dim, embed_dim), init=uniform(1)) + # TODO: Use Golve to initialize the embedding + # TODO: Add dropout to embedding query_embedding = times(query_sequence , embedding) context_embedding = times(context_sequence, embedding) @@ -244,7 +226,6 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor candidate_memory = sequence.scatter(sequence.gather(context_memory, candidate_filter, name='Candidate_Mem'), candidate_sc) candidate_ids = sequence.scatter(sequence.gather(candidate_indicates, candidate_filter, name = 'Candidate_Ids'), candidate_sc) entity_raw_ids = sequence.scatter(sequence.gather(reshape(context_raw, vocab_dim), candidate_filter), candidate_sc) - qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis query_memory = splice((qfwd, qbwd), name='Query_SP') # get the source (aka 'query') representation @@ -298,12 +279,92 @@ def loss(model): item_labels = sequence.reduce_sum(times(reshape(labels, (1,)), entity_ids), name='item_labels') mask = sequence.reduce_sum(entity_ids) cross_entroy = mask_cross_entropy(item_preds, item_labels, mask, name='CrossEntropyLoss') - return combine([cross_entroy, answers, labels, item_preds]) + probs = ops.element_select(mask, ops.exp(item_preds), 0, name='item_probs') + apply_loss = combine([cross_entroy, answers, labels, item_preds, probs]) + return Block(apply_loss, 'AvgSoftMaxCrossEntropy', Record(labels=item_labels)) -def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - entities = StreamDef(field='E', shape=1, is_sparse=False), - label = StreamDef(field='L', shape=1, is_sparse=False) - )), randomize=randomize, epoch_size = size) +#TODO: Add AUC for evaluation + +def train(model, reader, max_epochs=1, save_model_flag=False, epoch_size=270000): + # Criterion nodes + criterion_loss = loss(model) + loss_func = criterion_loss.outputs[0] + eval_func = classification_error(criterion_loss.outputs[-1], criterion_loss.labels) + + # Instantiate the trainer object to drive the model training + learning_rate = 0.005 + lr_per_sample = learning_rate_schedule(learning_rate, UnitType.minibatch) + + #minibatch_size = 30000 # max(sequence_length) --> so with avg length of context=1000 this is like 30 "full samples" + minibatch_size = 5000 + + momentum_time_constant = momentum_as_time_constant_schedule(1100) + clipping_threshold_per_sample = 10.0 + gradient_clipping_with_truncation = True + learner = momentum_sgd(model.parameters, + lr_per_sample, momentum_time_constant, + gradient_clipping_threshold_per_sample=clipping_threshold_per_sample, + gradient_clipping_with_truncation=gradient_clipping_with_truncation) + trainer = Trainer(model.outputs[-1], loss_func, eval_func, learner) + + # Get minibatches of sequences to train with and perform model training + i = 0 + mbs = 0 + #epoch_size = 270000 # this number is in sequences -- need to fix (unfortunately has to be in 'elements' for now) + # for ^^, we just need to keep adding up all the samples (1 per sequence) and end the epoch once we get to 270000 + training_progress_output_freq = 1 + + # bind inputs to data from readers + data_bind = {} + label_key = None + for arg in criterion_loss.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + if arg.name == 'labels': + label_key = arg + data_bind[arg] = reader.streams.label + + for epoch in range(max_epochs): + loss_numer = 0 + metric_numer = 0 + denom = 0 + + while i < (epoch+1) * epoch_size: + + # get next minibatch of training data + #mb_train = train_reader.next_minibatch(minibatch_size_in_samples=minibatch_size, input_map=train_bind) + # TODO: When will next_minibatch ended? + # TODO: Shuffle entities? @yelong + mb_train = reader.next_minibatch(1024, input_map=data_bind) + trainer.train_minibatch(mb_train) + + # collect epoch-wide stats + samples = trainer.previous_minibatch_sample_count + loss_numer += trainer.previous_minibatch_loss_average * samples + metric_numer += trainer.previous_minibatch_evaluation_average * samples + denom += samples + + # debugging + #print("previous minibatch sample count = %d" % samples) + #print("mb_train[labels] num samples = %d" % mb_train[labels].num_samples) + #print("previous minibatch loss average = %f" % trainer.previous_minibatch_loss_average) + + if mbs % training_progress_output_freq == 0: + print("Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}".format(mbs, + get_train_loss(trainer), get_train_eval_criterion(trainer))) + print("previous minibatch sample count = %d" % samples) + + i += mb_train[label_key].num_samples + mbs += 1 + + print("--- EPOCH %d DONE: loss = %f, errs = %f ---" % (epoch, loss_numer/denom, 100.0*(metric_numer/denom))) + + if save_model_flag: + # save the model every epoch + model_filename = os.path.join('model', "model_epoch%d.dnn" % epoch) + model.save_model(model_filename) + print("Saved model to '%s'" % model_filename) diff --git a/Examples/ReasoNet/tests/testReasoNetModel.py b/Examples/ReasoNet/tests/testReasoNetModel.py index c8e1fff71ea2..5a6046c43da0 100644 --- a/Examples/ReasoNet/tests/testReasoNetModel.py +++ b/Examples/ReasoNet/tests/testReasoNetModel.py @@ -2,6 +2,7 @@ import os import numpy as np from ReasoNet.model import create_model, create_reader, gru_cell +import ReasoNet.model as rsn import ReasoNet.asr as asr from cntk.blocks import Placeholder, Constant,initial_state_default_or_None, _is_given, _get_current_default_options from cntk.ops import input_variable, past_value, future_value @@ -21,13 +22,69 @@ def testModel(data): data_bind[arg] = reader.streams.context if arg.name == 'entities': data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(1, data_bind) + batch = reader.next_minibatch(2, data_bind) var = model.eval(batch) for o in model.outputs: print('-----------------------') print(o.name) print(np.around(var[o], decimals=3)) +def testReasoNetLoss(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + loss = rsn.loss(model) + data_bind = {} + for arg in loss.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + if arg.name == 'labels': + data_bind[arg] = reader.streams.label + batch = reader.next_minibatch(100, data_bind) + var = loss.eval(batch) + for o in loss.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + #pred = var[loss.outputs[-1]] + #for i in pred: + # print("Prediction: {0}".format(np.argmin(np.reshape(i, 10)))) + +def testReasoNetPred(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + pred = rsn.pred(model) + data_bind = {} + for arg in pred.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + if arg.name == 'labels': + data_bind[arg] = reader.streams.label + #data_bind = {model.context:reader.streams.context, model.query:reader.streams.query, model.entities:reader.streams.entities} + batch = reader.next_minibatch(100, data_bind) + var = pred.eval(batch) + for o in pred.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + pred = var[pred.outputs[-1]] + print('-----------------------') + for i in pred: + print('') + print("Prediction: {0}\n\t=>{1}\n".format(i, np.argmax(np.reshape(i, 10)))) + +def testReasoNetTrain(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + rsn.train(model, reader, epoch_size=10) + def testASR(data): reader = asr.create_reader(data, 10, False) model = asr.create_model(10, 5, 1) @@ -39,8 +96,9 @@ def testASR(data): data_bind[arg] = reader.streams.context if arg.name == 'entities': data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(1, data_bind) + batch = reader.next_minibatch(2, data_bind) var = model.eval(batch) + for o in model.outputs: print('-----------------------') print(o.name) @@ -61,4 +119,7 @@ def testGRU(): #testGRU() #testASR("test.idx") -testModel("test.idx") +#testModel("test.idx") +#testReasoNetLoss("test.idx") +testReasoNetTrain("test.idx") +#testReasoNetPred("test.idx") From 1e0f8629caa1a99c1ffbb3418b60e2a25ab640d0 Mon Sep 17 00:00:00 2001 From: PengchengHe Date: Fri, 6 Jan 2017 14:27:17 +0000 Subject: [PATCH 042/120] Fix training issues with ReasoNet --- Examples/ReasoNet/model.py | 214 +++++++++---------- Examples/ReasoNet/testReasoNetModel.py | 119 ----------- Examples/ReasoNet/testTextConvolution.py | 13 -- Examples/ReasoNet/tests/testReasoNetModel.py | 40 +++- Examples/ReasoNet/wordvocab.py | 4 +- bindings/python/setup.py | 2 +- 6 files changed, 143 insertions(+), 249 deletions(-) delete mode 100644 Examples/ReasoNet/testReasoNetModel.py delete mode 100644 Examples/ReasoNet/testTextConvolution.py diff --git a/Examples/ReasoNet/model.py b/Examples/ReasoNet/model.py index e0942299ebe0..667a5a3491c2 100644 --- a/Examples/ReasoNet/model.py +++ b/Examples/ReasoNet/model.py @@ -7,7 +7,6 @@ import numpy as np from cntk import Trainer, Axis, device, combine from cntk.io import MinibatchSource, CTFDeserializer, StreamDef, StreamDefs, INFINITELY_REPEAT -from cntk.learner import momentum_sgd, momentum_as_time_constant_schedule, learning_rate_schedule, UnitType from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, tanh, times_transpose, greater, cosine_distance, element_divide, element_select import cntk.ops as ops @@ -16,6 +15,8 @@ from cntk.initializer import uniform, glorot_uniform from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input, value_to_seq from cntk.utils.debughelpers import _name_node, _node_name, _node_description, _log_node +import cntk.utils as utils +import cntk.learner as learner ######################## @@ -147,8 +148,11 @@ def broadcast_as(op, seq_op, name=''): return combine([rlt], name=name) def cosine_similarity(src, tgt, name=''): - src_br = broadcast_as(src, tgt, name='cos_br') - sim = cosine_distance(src_br, tgt, name=name) + src_br = sequence.broadcast_as(src, tgt, name='cos_br') + dot = ops.times_transpose(src_br, tgt) + src_norm = ops.sqrt(ops.reduce_sum(ops.square(src_br))) + tgt_norm = ops.sqrt(ops.reduce_sum(ops.square(tgt))) + sim = ops.element_divide(dot, (src_norm*tgt_norm), name=name) return sim def project_cosine_sim(status, memory, dim, init = init_default_or_glorot_uniform, name=''): @@ -163,56 +167,35 @@ def termination_gate(status, dim, init = init_default_or_glorot_uniform, name='' Wt = Parameter((dim, 1), init = init, name='Wt') return sigmoid(times(status, Wt), name=name) -def attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids,hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): +def attention_rlunit(context_memory, query_memory, entity_memory, hidden_dim, init = init_default_or_glorot_uniform): status = Placeholder(name='status', shape=hidden_dim) context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim, name='context_attention') query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim, name='query_attetion') - context_attention = sequence.reduce_sum(element_times(context_attention_weight, context_memory), name='C-Att') - query_attention = sequence.reduce_sum(element_times(query_attention_weight, query_memory), name='Q-Att') + context_attention = sequence.reduce_sum(times(context_attention_weight, context_memory), name='C-Att') + query_attention = sequence.reduce_sum(times(query_attention_weight, query_memory), name='Q-Att') attention = splice((query_attention, context_attention), name='att-sp') - gru = gru_cell((hidden_dim, ), name='status') + gru = gru_cell((hidden_dim, ), name='control_status') new_status = gru(attention, status).output - termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') - ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim, name='ans_attention') - answers = times(ans_attention, candidate_ids, name='answers') - return combine([answers, termination_prob, new_status], name='ReinforcementAttention') + termination_prob = termination_gate(new_status, dim=hidden_dim, name='terminate_prob') + ans_attention = project_cosine_sim(new_status, entity_memory, hidden_dim, name='ans_attention') + return combine([ans_attention, termination_prob, new_status], name='ReinforcementAttention') -def seq_cross_entropy(pred, label, gama=10, name=''): - pred_exp = ops.exp(pred, name='pred_exp') - sum_exp = sequence.reduce_sum(pred_exp, name='sum_exp') - pred_sum = element_divide(pred_exp, sequence.broadcast_as(sum_exp, pred), name='exp_divid') - log_pred_sum = ops.log(pred_sum, name='log_pred') - label_pred = times(label, log_pred_sum, name = 'label_softmax') - entropy = ops.negate(sequence.reduce_sum(label_pred, name='sum_log'), name=name) - #loss = ops.negate(sequence.reduce_sum(times(label, ops.log(pred_exp/(sequence.broadcast_as(sum_exp, pred))))), name = name) - return entropy - -def mask_cross_entropy(pred, label, mask, gama=10, name=''): - pred_exp = element_select(mask, ops.exp(gama*pred), 0) - label_msk = element_select(label, 1, 0) - sum_exp = ops.reduce_sum(pred_exp) - soft_max = ops.element_select(mask, ops.negate(ops.element_times(label_msk, ops.log(pred_exp/sum_exp))), 0) - return ops.reduce_sum(soft_max, name=name) - -def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glorot_uniform): +# +# TODO: CNTK current will convert sparse variable to dense after reshape function +def create_model(vocab_dim, hidden_dim, embedded_dim=100, max_rl_iter=5, init=init_default_or_glorot_uniform): # Query and Doc/Context/Paragraph inputs to the model batch_axis = Axis.default_batch_axis() query_seq_axis = Axis('sourceAxis') context_seq_axis = Axis('contextAxis') query_dynamic_axes = [batch_axis, query_seq_axis] - query_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=query_dynamic_axes, name='query') + query_sequence = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=query_dynamic_axes, name='query') context_dynamic_axes = [batch_axis, context_seq_axis] - context_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') + context_sequence = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') candidate_dynamic_axes = [batch_axis, context_seq_axis] - candidate_indicates = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') + entity_ids_mask = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') - # Query sequences - query_sequence = query_raw - # Doc/Context sequences - context_sequence = context_raw # embedding - embed_dim = hidden_dim - embedding = parameter(shape=(vocab_dim, embed_dim), init=uniform(1)) + embedding = parameter(shape=(vocab_dim, embedded_dim), init=uniform(1)) # TODO: Use Golve to initialize the embedding # TODO: Add dropout to embedding @@ -221,23 +204,23 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor # get source and context representations context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis - candidate_filter = greater(candidate_indicates, 0) - candidate_sc = sequence.gather(candidate_filter, candidate_filter) - candidate_memory = sequence.scatter(sequence.gather(context_memory, candidate_filter, name='Candidate_Mem'), candidate_sc) - candidate_ids = sequence.scatter(sequence.gather(candidate_indicates, candidate_filter, name = 'Candidate_Ids'), candidate_sc) - entity_raw_ids = sequence.scatter(sequence.gather(reshape(context_raw, vocab_dim), candidate_filter), candidate_sc) + entity_condition = greater(entity_ids_mask, 0) + entities_all = sequence.gather(entity_condition, entity_condition) + entity_memory = sequence.scatter(sequence.gather(context_memory, entity_condition, name='Candidate_Mem'), entities_all) qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis query_memory = splice((qfwd, qbwd), name='Query_SP') # get the source (aka 'query') representation - status = splice((sequence.last(qfwd), sequence.first(qbwd)), name='Init_Status') # get last fwd status and first bwd status - attention_rlu = attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids, hidden_dim*2, vocab_dim, init) + init_status = splice((sequence.last(qfwd), sequence.first(qbwd)), name='Init_Status') # get last fwd status and first bwd status + attention_rlu = attention_rlunit(context_memory, query_memory, entity_memory, hidden_dim*2, init) status_controls = [] arlus = [None] * max_rl_iter answers = None probs = None for i in range(0, max_rl_iter): - arlus[i] = attention_rlu(status) - status = arlus[i].outputs[2] + if i == 0: + arlus[i] = attention_rlu(init_status) + else: + arlus[i] = attention_rlu(arlus[i-1].outputs[2]) status_controls += list(arlus[i].outputs[0:2]) if answers == None: answers = element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) @@ -245,75 +228,96 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor else: answers += element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) probs += arlus[i].outputs[1] - final_answers = element_divide(answers, sequence.broadcast_as(probs, answers), name='final_answers') - result = combine(status_controls+[final_answers], name='ReasoNet') - return Block(result, 'ReasoNet', Record(vocab_dim=vocab_dim, hidden_dim=hidden_dim, max_ite =max_rl_iter, context=context_raw, - query=query_raw, entities=candidate_indicates, entity_masks=candidate_filter, - entity_seqs=candidate_sc, entity_ids=entity_raw_ids)) + final_answers = reshape(element_divide(answers, sequence.broadcast_as(probs, answers)), (1,), name='final_answers') + result = combine([final_answers], name='ReasoNet') + #result = combine(status_controls+[final_answers], name='ReasoNet') + return Block(result, 'ReasoNet', Record(vocab_dim=vocab_dim, hidden_dim=hidden_dim, max_iter =max_rl_iter, context=context_sequence, + query=query_sequence, entities=entity_ids_mask, entity_condition=entity_condition, + entities_all=entities_all)) def pred(model): context = model.context entities = model.entities wordvocab_dim = model.vocab_dim - candidate_filter = model.entity_masks - candidate_sc = model.entity_seqs - answers = sequence.scatter(model.outputs[-1], candidate_sc, name='answers_prob') - entity_ids = model.entity_ids + entity_condition = model.entity_condition + entities_all = model.entities_all + answers = sequence.scatter(model.outputs[-1], entities_all, name='answers_prob') + entity_ids = sequence.scatter(sequence.gather(reshape(model.context, wordvocab_dim), entity_condition), entities_all) item_preds = sequence.reduce_sum(times(reshape(answers, (1,)), entity_ids), name = 'item_preds') mask = sequence.reduce_sum(entity_ids, name='mask') probs = ops.element_select(mask, ops.exp(item_preds), 0, name='item_prob') return combine([mask, probs]) +def accuracy_func(pred, label, name='accuracy'): + pred_max = ops.hardmax(pred, name='pred_max') + norm_label = ops.equal(label, [1], name='norm_label') + acc = ops.times_transpose(pred_max, norm_label, name='accuracy') + return acc + +def seq_accuracy(pred, label, name=''): + m = placeholder_variable(shape=(1,), dynamic_axes = pred.dynamic_axes, name='max') + o = element_select(greater(pred, past_value(m)), pred, past_value(m)) + rlt = o.replace_placeholders({m:sanitize_input(o)}) + max_val = sequence.broadcast_as(sequence.last(rlt), rlt) + first_max = sequence.first(sequence.where(ops.greater_equal(pred, max_val))) + label_idx = sequence.first(sequence.where(ops.equal(label, 1))) + return ops.equal(first_max, label_idx, name=name) + +def seq_cross_entropy(pred, label, gama=10, name=''): + #loss = ops.negate(sequence.reduce_sum(times(label, ops.log(pred_exp/(sequence.broadcast_as(sum_exp, pred))))), name = name) + pred_exp = ops.exp(pred*gama, name='pred_exp') + sum_exp = sequence.reduce_sum(pred_exp, name='sum_exp') + pred_prob = element_divide(pred_exp, sequence.broadcast_as(sum_exp, pred), name='prob') + log_prob = ops.log(pred_prob, name='log_prob') + label_softmax = ops.element_times(label, log_prob, name = 'label_softmax') + entropy = ops.negate(sequence.reduce_sum(label_softmax), name=name) + accuracy = seq_accuracy(pred_prob, label, name='accuracy') + return (entropy, accuracy) + +def mask_cross_entropy(pred, label, mask, gama=10, name=''): + pred_exp = element_select(mask, ops.exp(gama*pred), 0) + label_msk = element_select(label, 1, 0) + sum_exp = ops.reduce_sum(pred_exp) + soft_max = ops.element_select(mask, ops.negate(ops.element_times(label_msk, ops.log(pred_exp/sum_exp))), 0) + return ops.reduce_sum(soft_max, name=name) + def loss(model): context = model.context entities = model.entities wordvocab_dim = model.vocab_dim labels_raw = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context.dynamic_axes, name='labels') - candidate_filter = model.entity_masks - candidate_sc = model.entity_seqs - answers = sequence.scatter(model.outputs[-1], candidate_sc, name='answers_prob') - entity_ids = model.entity_ids - item_preds = sequence.reduce_sum(times(reshape(answers, (1,)), entity_ids), name = 'item_preds') - labels = sequence.scatter(sequence.gather(labels_raw, candidate_filter, name='EntityLabels'), candidate_sc, name='seq_labels') - #cross_entroy = seq_cross_entroy(reshape(answers, (1,)), labels, name='CrossEntropyLoss') - item_labels = sequence.reduce_sum(times(reshape(labels, (1,)), entity_ids), name='item_labels') - mask = sequence.reduce_sum(entity_ids) - cross_entroy = mask_cross_entropy(item_preds, item_labels, mask, name='CrossEntropyLoss') - probs = ops.element_select(mask, ops.exp(item_preds), 0, name='item_probs') - apply_loss = combine([cross_entroy, answers, labels, item_preds, probs]) - return Block(apply_loss, 'AvgSoftMaxCrossEntropy', Record(labels=item_labels)) - -#TODO: Add AUC for evaluation + entity_condition = model.entity_condition + entities_all = model.entities_all + answers = model.outputs[-1] + labels = sequence.scatter(sequence.gather(labels_raw, entity_condition, name='EntityLabels'), entities_all, name='seq_labels') + cross_entroy, accuracy = seq_cross_entropy(answers, labels, name='CrossEntropyLoss') + apply_loss = combine([cross_entroy, answers, labels, accuracy]) + return Block(apply_loss, 'AvgSoftMaxCrossEntropy', Record(labels=labels_raw)) def train(model, reader, max_epochs=1, save_model_flag=False, epoch_size=270000): # Criterion nodes criterion_loss = loss(model) loss_func = criterion_loss.outputs[0] - eval_func = classification_error(criterion_loss.outputs[-1], criterion_loss.labels) + eval_func = criterion_loss.outputs[-1] + #eval_func = accuracy_func(criterion_loss.outputs[-1], criterion_loss.labels) # Instantiate the trainer object to drive the model training learning_rate = 0.005 - lr_per_sample = learning_rate_schedule(learning_rate, UnitType.minibatch) - - #minibatch_size = 30000 # max(sequence_length) --> so with avg length of context=1000 this is like 30 "full samples" - minibatch_size = 5000 - - momentum_time_constant = momentum_as_time_constant_schedule(1100) + lr_schedule = learner.learning_rate_schedule(learning_rate, learner.UnitType.minibatch) + minibatch_size = 1024*12 + momentum = learner.momentum_schedule(0.9) + momentum_var = learner.momentum_schedule(0.999) clipping_threshold_per_sample = 10.0 gradient_clipping_with_truncation = True - learner = momentum_sgd(model.parameters, - lr_per_sample, momentum_time_constant, - gradient_clipping_threshold_per_sample=clipping_threshold_per_sample, - gradient_clipping_with_truncation=gradient_clipping_with_truncation) - trainer = Trainer(model.outputs[-1], loss_func, eval_func, learner) + #learn = learner.adam_sgd(model.parameters, lr_schedule, momentum, momentum_var, + # gradient_clipping_threshold_per_sample=clipping_threshold_per_sample, + # gradient_clipping_with_truncation=gradient_clipping_with_truncation) + learn = learner.momentum_sgd(model.parameters, lr_schedule, momentum, + gradient_clipping_threshold_per_sample=clipping_threshold_per_sample, + gradient_clipping_with_truncation=gradient_clipping_with_truncation) + trainer = Trainer(model.outputs[-1], loss_func, eval_func, learn) # Get minibatches of sequences to train with and perform model training - i = 0 - mbs = 0 - #epoch_size = 270000 # this number is in sequences -- need to fix (unfortunately has to be in 'elements' for now) - # for ^^, we just need to keep adding up all the samples (1 per sequence) and end the epoch once we get to 270000 - training_progress_output_freq = 1 - # bind inputs to data from readers data_bind = {} label_key = None @@ -328,40 +332,36 @@ def train(model, reader, max_epochs=1, save_model_flag=False, epoch_size=270000) label_key = arg data_bind[arg] = reader.streams.label + i = 0 + minibatch_count = 0 + training_progress_output_freq = 100 + for epoch in range(max_epochs): loss_numer = 0 metric_numer = 0 - denom = 0 + total_samples = 0 while i < (epoch+1) * epoch_size: - # get next minibatch of training data - #mb_train = train_reader.next_minibatch(minibatch_size_in_samples=minibatch_size, input_map=train_bind) - # TODO: When will next_minibatch ended? # TODO: Shuffle entities? @yelong - mb_train = reader.next_minibatch(1024, input_map=data_bind) + mb_train = reader.next_minibatch(minibatch_size, input_map=data_bind) + trainer.train_minibatch(mb_train) # collect epoch-wide stats samples = trainer.previous_minibatch_sample_count loss_numer += trainer.previous_minibatch_loss_average * samples metric_numer += trainer.previous_minibatch_evaluation_average * samples - denom += samples - - # debugging - #print("previous minibatch sample count = %d" % samples) - #print("mb_train[labels] num samples = %d" % mb_train[labels].num_samples) - #print("previous minibatch loss average = %f" % trainer.previous_minibatch_loss_average) - - if mbs % training_progress_output_freq == 0: - print("Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}".format(mbs, - get_train_loss(trainer), get_train_eval_criterion(trainer))) - print("previous minibatch sample count = %d" % samples) + total_samples += samples + if int(total_samples/training_progress_output_freq) != int((total_samples-samples)/training_progress_output_freq): + print("Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}".format(minibatch_count, + get_train_loss(trainer), get_train_eval_criterion(trainer))) + print("Total sample count = {}, previous minibatch size={}".format(total_samples, samples)) i += mb_train[label_key].num_samples - mbs += 1 + minibatch_count += 1 - print("--- EPOCH %d DONE: loss = %f, errs = %f ---" % (epoch, loss_numer/denom, 100.0*(metric_numer/denom))) + print("--- EPOCH %d DONE: loss = %f, errs = %.2f%% ---" % (epoch, loss_numer/total_samples, 100.0*(metric_numer/total_samples))) if save_model_flag: # save the model every epoch diff --git a/Examples/ReasoNet/testReasoNetModel.py b/Examples/ReasoNet/testReasoNetModel.py deleted file mode 100644 index 3513f17a6fa9..000000000000 --- a/Examples/ReasoNet/testReasoNetModel.py +++ /dev/null @@ -1,119 +0,0 @@ -import sys -import os -import numpy as np -from ReasoNet.model import create_model, create_reader, gru_cell -import ReasoNet.model as rsn -import ReasoNet.asr as asr -from cntk.blocks import Placeholder, Constant,initial_state_default_or_None, _is_given, _get_current_default_options -from cntk.ops import input_variable, past_value, future_value -from cntk.io import MinibatchSource -from cntk import Trainer, Axis, device, combine -from cntk.layers import Recurrence, Convolution -from cntk.utils import _as_tuple - -def testModel(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - data_bind = {} - for arg in model.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(2, data_bind) - var = model.eval(batch) - for o in model.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - -def testReasoNetLoss(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - loss = rsn.loss(model) - data_bind = {} - for arg in loss.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - if arg.name == 'labels': - data_bind[arg] = reader.streams.label - batch = reader.next_minibatch(100, data_bind) - var = loss.eval(batch) - for o in loss.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - #pred = var[loss.outputs[-1]] - #for i in pred: - # print("Prediction: {0}".format(np.argmin(np.reshape(i, 10)))) - -def testReasoNetPred(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - pred = rsn.pred(model) - data_bind = {} - for arg in pred.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - if arg.name == 'labels': - data_bind[arg] = reader.streams.label - #data_bind = {model.context:reader.streams.context, model.query:reader.streams.query, model.entities:reader.streams.entities} - batch = reader.next_minibatch(100, data_bind) - var = pred.eval(batch) - for o in pred.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - pred = var[pred.outputs[-1]] - print('-----------------------') - for i in pred: - print('') - print("Prediction: {0}\n\t=>{1}\n".format(i, np.argmax(np.reshape(i, 10)))) - -def testASR(data): - reader = asr.create_reader(data, 10, False) - model = asr.create_model(10, 5, 1) - data_bind = {} - for arg in model.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(2, data_bind) - var = model.eval(batch) - - for o in model.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - - -def testGRU(): - g = gru_cell(5) - x = np.reshape(np.arange(0,25, dtype=np.float32), (1,5,5)) - a = input_variable(shape=(5,), dynamic_axes=[Axis.default_batch_axis(), Axis('Seq')]) - y = np.float32([1,2,0.1,0.2,1]) - s = Constant(y) - q = g(a,s).eval({a:x}) - print(q) - r = Recurrence(gru_cell(5)) - rt = r(a).eval({a:x}) - print(rt) - -#testGRU() -#testASR("test.idx") -#testModel("test.idx") -#testReasoNetLoss("test.idx") -testReasoNetPred("test.idx") diff --git a/Examples/ReasoNet/testTextConvolution.py b/Examples/ReasoNet/testTextConvolution.py deleted file mode 100644 index db62f28a17e3..000000000000 --- a/Examples/ReasoNet/testTextConvolution.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys -import os -import numpy as np -from ReasoNet.model import text_convolution -from cntk.blocks import Placeholder, Constant -from cntk.ops import input_variable - -def testTextConvolution(): - text = np.reshape(np.arange(25.0, dtype=np.float32), (1, 5,5)) - x = input_variable(shape=(1, 5, 5,)) - c = text_convolution(3, 5, 5)(x) - v = c.eval([text])[0] - print(v) diff --git a/Examples/ReasoNet/tests/testReasoNetModel.py b/Examples/ReasoNet/tests/testReasoNetModel.py index 5a6046c43da0..c54d83d7d4b0 100644 --- a/Examples/ReasoNet/tests/testReasoNetModel.py +++ b/Examples/ReasoNet/tests/testReasoNetModel.py @@ -1,5 +1,7 @@ import sys import os +import cntk.device as device +#device.set_default_device(device.cpu()) import numpy as np from ReasoNet.model import create_model, create_reader, gru_cell import ReasoNet.model as rsn @@ -10,6 +12,8 @@ from cntk import Trainer, Axis, device, combine from cntk.layers import Recurrence, Convolution from cntk.utils import _as_tuple +import cntk.ops as ops +import cntk def testModel(data): reader = create_reader(data, 10, False) @@ -30,8 +34,8 @@ def testModel(data): print(np.around(var[o], decimals=3)) def testReasoNetLoss(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) + reader = create_reader(data, 101000, False) + model = create_model(101000, 300, 3) loss = rsn.loss(model) data_bind = {} for arg in loss.arguments: @@ -80,10 +84,10 @@ def testReasoNetPred(data): print('') print("Prediction: {0}\n\t=>{1}\n".format(i, np.argmax(np.reshape(i, 10)))) -def testReasoNetTrain(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - rsn.train(model, reader, epoch_size=10) +def testReasoNetTrain(data, epoch_size, max_epochs=1, vocab_dim=101000, hidden_dim=300, max_rl_iter =5): + reader = create_reader(data, vocab_dim, True) + model = create_model(vocab_dim, hidden_dim, embedded_dim=100, max_rl_iter=max_rl_iter) + rsn.train(model, reader, max_epochs=max_epochs, epoch_size=epoch_size) def testASR(data): reader = asr.create_reader(data, 10, False) @@ -117,9 +121,31 @@ def testGRU(): rt = r(a).eval({a:x}) print(rt) +def testSparse(data, vocab_dim=101000, hidden_dim=300, max_rl_iter =5): + reader = create_reader(data, vocab_dim, False) + data = reader.next_minibatch(1) + context_data = data[reader.streams.context] + context_dynamic_axes = [cntk.Axis.default_batch_axis(), cntk.Axis('Context')] + context_var= input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') + q = ops.times(1, context_var) + print(q.output.is_sparse) + su = ops.sequence.reduce_sum(ops.reshape(context_var, vocab_dim)) + o = su.eval({context_var:context_data}) + print(o) + #testGRU() #testASR("test.idx") #testModel("test.idx") #testReasoNetLoss("test.idx") -testReasoNetTrain("test.idx") +#testReasoNetLoss("test.idx") +#testReasoNetTrain("test.idx", 10, vocab_dim=10, hidden_dim=5, max_rl_iter=3) #testReasoNetPred("test.idx") + +#testReasoNetTrain("test.5.idx", 5968, max_epochs=5, vocab_dim=101000, hidden_dim=300, max_rl_iter=5) +#testReasoNetTrain("test.7.idx", 7394, max_epochs=5, vocab_dim=101000, hidden_dim=300, max_rl_iter=5) +#testReasoNetTrain("test.10.idx", 11238, max_epochs=5, vocab_dim=101000, hidden_dim=300, max_rl_iter=5) +testReasoNetTrain("test.1000.idx", 1168157, max_epochs=5, vocab_dim=101000, hidden_dim=384, max_rl_iter=5) +#testReasoNetTrain("test.1000.40k.idx", 1168157, max_epochs=5, vocab_dim=40000, hidden_dim=384, max_rl_iter=5) + + +#testSparse("test.idx", 101000) diff --git a/Examples/ReasoNet/wordvocab.py b/Examples/ReasoNet/wordvocab.py index d1ac34eb3b73..3bac68eeb4ae 100644 --- a/Examples/ReasoNet/wordvocab.py +++ b/Examples/ReasoNet/wordvocab.py @@ -189,9 +189,9 @@ def build_bing_corpus_index(entities, words, corpus, index): index.write("{0}".format(seq_id)) for i in range(max(len(context_ids), len(query_ids))): if i < len(query_ids): - index.write(" |Q {0}".format(query_ids[i])) + index.write(" |Q {0}:1".format(query_ids[i])) if i < len(context_ids): - index.write(" |C {0}".format(context_ids[i])) + index.write(" |C {0}:1".format(context_ids[i])) index.write(" |E {0}".format(is_entity[i])) index.write(" |L {0}".format(labels[i])) index.write("\n") diff --git a/bindings/python/setup.py b/bindings/python/setup.py index d771f124a249..e1187cfb6bc7 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -42,7 +42,7 @@ CNTK_LIB_PATH = os.path.join(CNTK_PATH, "x64", "Release") else: CNTK_LIB_PATH = os.path.join( - CNTK_PATH, "build", "gpu", "debug", "lib") + CNTK_PATH, "build", "gpu", "release", "lib") print("Using CNTK sources at '%s'" % os.path.abspath(CNTK_SOURCE_PATH)) print("Using CNTK libs at '%s'" % os.path.abspath(CNTK_LIB_PATH)) From 510595348fde47e635cf44fc70558b5ad5618164 Mon Sep 17 00:00:00 2001 From: PengchengHe Date: Fri, 6 Jan 2017 14:51:42 +0000 Subject: [PATCH 043/120] Revert "Fix training issues with ReasoNet" This reverts commit 1e0f8629caa1a99c1ffbb3418b60e2a25ab640d0. --- Examples/ReasoNet/model.py | 214 +++++++++---------- Examples/ReasoNet/testReasoNetModel.py | 119 +++++++++++ Examples/ReasoNet/testTextConvolution.py | 13 ++ Examples/ReasoNet/tests/testReasoNetModel.py | 40 +--- Examples/ReasoNet/wordvocab.py | 4 +- bindings/python/setup.py | 2 +- 6 files changed, 249 insertions(+), 143 deletions(-) create mode 100644 Examples/ReasoNet/testReasoNetModel.py create mode 100644 Examples/ReasoNet/testTextConvolution.py diff --git a/Examples/ReasoNet/model.py b/Examples/ReasoNet/model.py index 667a5a3491c2..e0942299ebe0 100644 --- a/Examples/ReasoNet/model.py +++ b/Examples/ReasoNet/model.py @@ -7,6 +7,7 @@ import numpy as np from cntk import Trainer, Axis, device, combine from cntk.io import MinibatchSource, CTFDeserializer, StreamDef, StreamDefs, INFINITELY_REPEAT +from cntk.learner import momentum_sgd, momentum_as_time_constant_schedule, learning_rate_schedule, UnitType from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, tanh, times_transpose, greater, cosine_distance, element_divide, element_select import cntk.ops as ops @@ -15,8 +16,6 @@ from cntk.initializer import uniform, glorot_uniform from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input, value_to_seq from cntk.utils.debughelpers import _name_node, _node_name, _node_description, _log_node -import cntk.utils as utils -import cntk.learner as learner ######################## @@ -148,11 +147,8 @@ def broadcast_as(op, seq_op, name=''): return combine([rlt], name=name) def cosine_similarity(src, tgt, name=''): - src_br = sequence.broadcast_as(src, tgt, name='cos_br') - dot = ops.times_transpose(src_br, tgt) - src_norm = ops.sqrt(ops.reduce_sum(ops.square(src_br))) - tgt_norm = ops.sqrt(ops.reduce_sum(ops.square(tgt))) - sim = ops.element_divide(dot, (src_norm*tgt_norm), name=name) + src_br = broadcast_as(src, tgt, name='cos_br') + sim = cosine_distance(src_br, tgt, name=name) return sim def project_cosine_sim(status, memory, dim, init = init_default_or_glorot_uniform, name=''): @@ -167,35 +163,56 @@ def termination_gate(status, dim, init = init_default_or_glorot_uniform, name='' Wt = Parameter((dim, 1), init = init, name='Wt') return sigmoid(times(status, Wt), name=name) -def attention_rlunit(context_memory, query_memory, entity_memory, hidden_dim, init = init_default_or_glorot_uniform): +def attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids,hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): status = Placeholder(name='status', shape=hidden_dim) context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim, name='context_attention') query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim, name='query_attetion') - context_attention = sequence.reduce_sum(times(context_attention_weight, context_memory), name='C-Att') - query_attention = sequence.reduce_sum(times(query_attention_weight, query_memory), name='Q-Att') + context_attention = sequence.reduce_sum(element_times(context_attention_weight, context_memory), name='C-Att') + query_attention = sequence.reduce_sum(element_times(query_attention_weight, query_memory), name='Q-Att') attention = splice((query_attention, context_attention), name='att-sp') - gru = gru_cell((hidden_dim, ), name='control_status') + gru = gru_cell((hidden_dim, ), name='status') new_status = gru(attention, status).output - termination_prob = termination_gate(new_status, dim=hidden_dim, name='terminate_prob') - ans_attention = project_cosine_sim(new_status, entity_memory, hidden_dim, name='ans_attention') - return combine([ans_attention, termination_prob, new_status], name='ReinforcementAttention') + termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') + ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim, name='ans_attention') + answers = times(ans_attention, candidate_ids, name='answers') + return combine([answers, termination_prob, new_status], name='ReinforcementAttention') -# -# TODO: CNTK current will convert sparse variable to dense after reshape function -def create_model(vocab_dim, hidden_dim, embedded_dim=100, max_rl_iter=5, init=init_default_or_glorot_uniform): +def seq_cross_entropy(pred, label, gama=10, name=''): + pred_exp = ops.exp(pred, name='pred_exp') + sum_exp = sequence.reduce_sum(pred_exp, name='sum_exp') + pred_sum = element_divide(pred_exp, sequence.broadcast_as(sum_exp, pred), name='exp_divid') + log_pred_sum = ops.log(pred_sum, name='log_pred') + label_pred = times(label, log_pred_sum, name = 'label_softmax') + entropy = ops.negate(sequence.reduce_sum(label_pred, name='sum_log'), name=name) + #loss = ops.negate(sequence.reduce_sum(times(label, ops.log(pred_exp/(sequence.broadcast_as(sum_exp, pred))))), name = name) + return entropy + +def mask_cross_entropy(pred, label, mask, gama=10, name=''): + pred_exp = element_select(mask, ops.exp(gama*pred), 0) + label_msk = element_select(label, 1, 0) + sum_exp = ops.reduce_sum(pred_exp) + soft_max = ops.element_select(mask, ops.negate(ops.element_times(label_msk, ops.log(pred_exp/sum_exp))), 0) + return ops.reduce_sum(soft_max, name=name) + +def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glorot_uniform): # Query and Doc/Context/Paragraph inputs to the model batch_axis = Axis.default_batch_axis() query_seq_axis = Axis('sourceAxis') context_seq_axis = Axis('contextAxis') query_dynamic_axes = [batch_axis, query_seq_axis] - query_sequence = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=query_dynamic_axes, name='query') + query_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=query_dynamic_axes, name='query') context_dynamic_axes = [batch_axis, context_seq_axis] - context_sequence = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') + context_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') candidate_dynamic_axes = [batch_axis, context_seq_axis] - entity_ids_mask = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') + candidate_indicates = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') + # Query sequences + query_sequence = query_raw + # Doc/Context sequences + context_sequence = context_raw # embedding - embedding = parameter(shape=(vocab_dim, embedded_dim), init=uniform(1)) + embed_dim = hidden_dim + embedding = parameter(shape=(vocab_dim, embed_dim), init=uniform(1)) # TODO: Use Golve to initialize the embedding # TODO: Add dropout to embedding @@ -204,23 +221,23 @@ def create_model(vocab_dim, hidden_dim, embedded_dim=100, max_rl_iter=5, init=in # get source and context representations context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis - entity_condition = greater(entity_ids_mask, 0) - entities_all = sequence.gather(entity_condition, entity_condition) - entity_memory = sequence.scatter(sequence.gather(context_memory, entity_condition, name='Candidate_Mem'), entities_all) + candidate_filter = greater(candidate_indicates, 0) + candidate_sc = sequence.gather(candidate_filter, candidate_filter) + candidate_memory = sequence.scatter(sequence.gather(context_memory, candidate_filter, name='Candidate_Mem'), candidate_sc) + candidate_ids = sequence.scatter(sequence.gather(candidate_indicates, candidate_filter, name = 'Candidate_Ids'), candidate_sc) + entity_raw_ids = sequence.scatter(sequence.gather(reshape(context_raw, vocab_dim), candidate_filter), candidate_sc) qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis query_memory = splice((qfwd, qbwd), name='Query_SP') # get the source (aka 'query') representation - init_status = splice((sequence.last(qfwd), sequence.first(qbwd)), name='Init_Status') # get last fwd status and first bwd status - attention_rlu = attention_rlunit(context_memory, query_memory, entity_memory, hidden_dim*2, init) + status = splice((sequence.last(qfwd), sequence.first(qbwd)), name='Init_Status') # get last fwd status and first bwd status + attention_rlu = attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids, hidden_dim*2, vocab_dim, init) status_controls = [] arlus = [None] * max_rl_iter answers = None probs = None for i in range(0, max_rl_iter): - if i == 0: - arlus[i] = attention_rlu(init_status) - else: - arlus[i] = attention_rlu(arlus[i-1].outputs[2]) + arlus[i] = attention_rlu(status) + status = arlus[i].outputs[2] status_controls += list(arlus[i].outputs[0:2]) if answers == None: answers = element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) @@ -228,96 +245,75 @@ def create_model(vocab_dim, hidden_dim, embedded_dim=100, max_rl_iter=5, init=in else: answers += element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) probs += arlus[i].outputs[1] - final_answers = reshape(element_divide(answers, sequence.broadcast_as(probs, answers)), (1,), name='final_answers') - result = combine([final_answers], name='ReasoNet') - #result = combine(status_controls+[final_answers], name='ReasoNet') - return Block(result, 'ReasoNet', Record(vocab_dim=vocab_dim, hidden_dim=hidden_dim, max_iter =max_rl_iter, context=context_sequence, - query=query_sequence, entities=entity_ids_mask, entity_condition=entity_condition, - entities_all=entities_all)) + final_answers = element_divide(answers, sequence.broadcast_as(probs, answers), name='final_answers') + result = combine(status_controls+[final_answers], name='ReasoNet') + return Block(result, 'ReasoNet', Record(vocab_dim=vocab_dim, hidden_dim=hidden_dim, max_ite =max_rl_iter, context=context_raw, + query=query_raw, entities=candidate_indicates, entity_masks=candidate_filter, + entity_seqs=candidate_sc, entity_ids=entity_raw_ids)) def pred(model): context = model.context entities = model.entities wordvocab_dim = model.vocab_dim - entity_condition = model.entity_condition - entities_all = model.entities_all - answers = sequence.scatter(model.outputs[-1], entities_all, name='answers_prob') - entity_ids = sequence.scatter(sequence.gather(reshape(model.context, wordvocab_dim), entity_condition), entities_all) + candidate_filter = model.entity_masks + candidate_sc = model.entity_seqs + answers = sequence.scatter(model.outputs[-1], candidate_sc, name='answers_prob') + entity_ids = model.entity_ids item_preds = sequence.reduce_sum(times(reshape(answers, (1,)), entity_ids), name = 'item_preds') mask = sequence.reduce_sum(entity_ids, name='mask') probs = ops.element_select(mask, ops.exp(item_preds), 0, name='item_prob') return combine([mask, probs]) -def accuracy_func(pred, label, name='accuracy'): - pred_max = ops.hardmax(pred, name='pred_max') - norm_label = ops.equal(label, [1], name='norm_label') - acc = ops.times_transpose(pred_max, norm_label, name='accuracy') - return acc - -def seq_accuracy(pred, label, name=''): - m = placeholder_variable(shape=(1,), dynamic_axes = pred.dynamic_axes, name='max') - o = element_select(greater(pred, past_value(m)), pred, past_value(m)) - rlt = o.replace_placeholders({m:sanitize_input(o)}) - max_val = sequence.broadcast_as(sequence.last(rlt), rlt) - first_max = sequence.first(sequence.where(ops.greater_equal(pred, max_val))) - label_idx = sequence.first(sequence.where(ops.equal(label, 1))) - return ops.equal(first_max, label_idx, name=name) - -def seq_cross_entropy(pred, label, gama=10, name=''): - #loss = ops.negate(sequence.reduce_sum(times(label, ops.log(pred_exp/(sequence.broadcast_as(sum_exp, pred))))), name = name) - pred_exp = ops.exp(pred*gama, name='pred_exp') - sum_exp = sequence.reduce_sum(pred_exp, name='sum_exp') - pred_prob = element_divide(pred_exp, sequence.broadcast_as(sum_exp, pred), name='prob') - log_prob = ops.log(pred_prob, name='log_prob') - label_softmax = ops.element_times(label, log_prob, name = 'label_softmax') - entropy = ops.negate(sequence.reduce_sum(label_softmax), name=name) - accuracy = seq_accuracy(pred_prob, label, name='accuracy') - return (entropy, accuracy) - -def mask_cross_entropy(pred, label, mask, gama=10, name=''): - pred_exp = element_select(mask, ops.exp(gama*pred), 0) - label_msk = element_select(label, 1, 0) - sum_exp = ops.reduce_sum(pred_exp) - soft_max = ops.element_select(mask, ops.negate(ops.element_times(label_msk, ops.log(pred_exp/sum_exp))), 0) - return ops.reduce_sum(soft_max, name=name) - def loss(model): context = model.context entities = model.entities wordvocab_dim = model.vocab_dim labels_raw = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context.dynamic_axes, name='labels') - entity_condition = model.entity_condition - entities_all = model.entities_all - answers = model.outputs[-1] - labels = sequence.scatter(sequence.gather(labels_raw, entity_condition, name='EntityLabels'), entities_all, name='seq_labels') - cross_entroy, accuracy = seq_cross_entropy(answers, labels, name='CrossEntropyLoss') - apply_loss = combine([cross_entroy, answers, labels, accuracy]) - return Block(apply_loss, 'AvgSoftMaxCrossEntropy', Record(labels=labels_raw)) + candidate_filter = model.entity_masks + candidate_sc = model.entity_seqs + answers = sequence.scatter(model.outputs[-1], candidate_sc, name='answers_prob') + entity_ids = model.entity_ids + item_preds = sequence.reduce_sum(times(reshape(answers, (1,)), entity_ids), name = 'item_preds') + labels = sequence.scatter(sequence.gather(labels_raw, candidate_filter, name='EntityLabels'), candidate_sc, name='seq_labels') + #cross_entroy = seq_cross_entroy(reshape(answers, (1,)), labels, name='CrossEntropyLoss') + item_labels = sequence.reduce_sum(times(reshape(labels, (1,)), entity_ids), name='item_labels') + mask = sequence.reduce_sum(entity_ids) + cross_entroy = mask_cross_entropy(item_preds, item_labels, mask, name='CrossEntropyLoss') + probs = ops.element_select(mask, ops.exp(item_preds), 0, name='item_probs') + apply_loss = combine([cross_entroy, answers, labels, item_preds, probs]) + return Block(apply_loss, 'AvgSoftMaxCrossEntropy', Record(labels=item_labels)) + +#TODO: Add AUC for evaluation def train(model, reader, max_epochs=1, save_model_flag=False, epoch_size=270000): # Criterion nodes criterion_loss = loss(model) loss_func = criterion_loss.outputs[0] - eval_func = criterion_loss.outputs[-1] - #eval_func = accuracy_func(criterion_loss.outputs[-1], criterion_loss.labels) + eval_func = classification_error(criterion_loss.outputs[-1], criterion_loss.labels) # Instantiate the trainer object to drive the model training learning_rate = 0.005 - lr_schedule = learner.learning_rate_schedule(learning_rate, learner.UnitType.minibatch) - minibatch_size = 1024*12 - momentum = learner.momentum_schedule(0.9) - momentum_var = learner.momentum_schedule(0.999) + lr_per_sample = learning_rate_schedule(learning_rate, UnitType.minibatch) + + #minibatch_size = 30000 # max(sequence_length) --> so with avg length of context=1000 this is like 30 "full samples" + minibatch_size = 5000 + + momentum_time_constant = momentum_as_time_constant_schedule(1100) clipping_threshold_per_sample = 10.0 gradient_clipping_with_truncation = True - #learn = learner.adam_sgd(model.parameters, lr_schedule, momentum, momentum_var, - # gradient_clipping_threshold_per_sample=clipping_threshold_per_sample, - # gradient_clipping_with_truncation=gradient_clipping_with_truncation) - learn = learner.momentum_sgd(model.parameters, lr_schedule, momentum, - gradient_clipping_threshold_per_sample=clipping_threshold_per_sample, - gradient_clipping_with_truncation=gradient_clipping_with_truncation) - trainer = Trainer(model.outputs[-1], loss_func, eval_func, learn) + learner = momentum_sgd(model.parameters, + lr_per_sample, momentum_time_constant, + gradient_clipping_threshold_per_sample=clipping_threshold_per_sample, + gradient_clipping_with_truncation=gradient_clipping_with_truncation) + trainer = Trainer(model.outputs[-1], loss_func, eval_func, learner) # Get minibatches of sequences to train with and perform model training + i = 0 + mbs = 0 + #epoch_size = 270000 # this number is in sequences -- need to fix (unfortunately has to be in 'elements' for now) + # for ^^, we just need to keep adding up all the samples (1 per sequence) and end the epoch once we get to 270000 + training_progress_output_freq = 1 + # bind inputs to data from readers data_bind = {} label_key = None @@ -332,36 +328,40 @@ def train(model, reader, max_epochs=1, save_model_flag=False, epoch_size=270000) label_key = arg data_bind[arg] = reader.streams.label - i = 0 - minibatch_count = 0 - training_progress_output_freq = 100 - for epoch in range(max_epochs): loss_numer = 0 metric_numer = 0 - total_samples = 0 + denom = 0 while i < (epoch+1) * epoch_size: + # get next minibatch of training data + #mb_train = train_reader.next_minibatch(minibatch_size_in_samples=minibatch_size, input_map=train_bind) + # TODO: When will next_minibatch ended? # TODO: Shuffle entities? @yelong - mb_train = reader.next_minibatch(minibatch_size, input_map=data_bind) - + mb_train = reader.next_minibatch(1024, input_map=data_bind) trainer.train_minibatch(mb_train) # collect epoch-wide stats samples = trainer.previous_minibatch_sample_count loss_numer += trainer.previous_minibatch_loss_average * samples metric_numer += trainer.previous_minibatch_evaluation_average * samples - total_samples += samples + denom += samples + + # debugging + #print("previous minibatch sample count = %d" % samples) + #print("mb_train[labels] num samples = %d" % mb_train[labels].num_samples) + #print("previous minibatch loss average = %f" % trainer.previous_minibatch_loss_average) + + if mbs % training_progress_output_freq == 0: + print("Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}".format(mbs, + get_train_loss(trainer), get_train_eval_criterion(trainer))) + print("previous minibatch sample count = %d" % samples) - if int(total_samples/training_progress_output_freq) != int((total_samples-samples)/training_progress_output_freq): - print("Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}".format(minibatch_count, - get_train_loss(trainer), get_train_eval_criterion(trainer))) - print("Total sample count = {}, previous minibatch size={}".format(total_samples, samples)) i += mb_train[label_key].num_samples - minibatch_count += 1 + mbs += 1 - print("--- EPOCH %d DONE: loss = %f, errs = %.2f%% ---" % (epoch, loss_numer/total_samples, 100.0*(metric_numer/total_samples))) + print("--- EPOCH %d DONE: loss = %f, errs = %f ---" % (epoch, loss_numer/denom, 100.0*(metric_numer/denom))) if save_model_flag: # save the model every epoch diff --git a/Examples/ReasoNet/testReasoNetModel.py b/Examples/ReasoNet/testReasoNetModel.py new file mode 100644 index 000000000000..3513f17a6fa9 --- /dev/null +++ b/Examples/ReasoNet/testReasoNetModel.py @@ -0,0 +1,119 @@ +import sys +import os +import numpy as np +from ReasoNet.model import create_model, create_reader, gru_cell +import ReasoNet.model as rsn +import ReasoNet.asr as asr +from cntk.blocks import Placeholder, Constant,initial_state_default_or_None, _is_given, _get_current_default_options +from cntk.ops import input_variable, past_value, future_value +from cntk.io import MinibatchSource +from cntk import Trainer, Axis, device, combine +from cntk.layers import Recurrence, Convolution +from cntk.utils import _as_tuple + +def testModel(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + data_bind = {} + for arg in model.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + batch = reader.next_minibatch(2, data_bind) + var = model.eval(batch) + for o in model.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + +def testReasoNetLoss(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + loss = rsn.loss(model) + data_bind = {} + for arg in loss.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + if arg.name == 'labels': + data_bind[arg] = reader.streams.label + batch = reader.next_minibatch(100, data_bind) + var = loss.eval(batch) + for o in loss.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + #pred = var[loss.outputs[-1]] + #for i in pred: + # print("Prediction: {0}".format(np.argmin(np.reshape(i, 10)))) + +def testReasoNetPred(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + pred = rsn.pred(model) + data_bind = {} + for arg in pred.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + if arg.name == 'labels': + data_bind[arg] = reader.streams.label + #data_bind = {model.context:reader.streams.context, model.query:reader.streams.query, model.entities:reader.streams.entities} + batch = reader.next_minibatch(100, data_bind) + var = pred.eval(batch) + for o in pred.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + pred = var[pred.outputs[-1]] + print('-----------------------') + for i in pred: + print('') + print("Prediction: {0}\n\t=>{1}\n".format(i, np.argmax(np.reshape(i, 10)))) + +def testASR(data): + reader = asr.create_reader(data, 10, False) + model = asr.create_model(10, 5, 1) + data_bind = {} + for arg in model.arguments: + if arg.name == 'query': + data_bind[arg] = reader.streams.query + if arg.name == 'context': + data_bind[arg] = reader.streams.context + if arg.name == 'entities': + data_bind[arg] = reader.streams.entities + batch = reader.next_minibatch(2, data_bind) + var = model.eval(batch) + + for o in model.outputs: + print('-----------------------') + print(o.name) + print(np.around(var[o], decimals=3)) + + +def testGRU(): + g = gru_cell(5) + x = np.reshape(np.arange(0,25, dtype=np.float32), (1,5,5)) + a = input_variable(shape=(5,), dynamic_axes=[Axis.default_batch_axis(), Axis('Seq')]) + y = np.float32([1,2,0.1,0.2,1]) + s = Constant(y) + q = g(a,s).eval({a:x}) + print(q) + r = Recurrence(gru_cell(5)) + rt = r(a).eval({a:x}) + print(rt) + +#testGRU() +#testASR("test.idx") +#testModel("test.idx") +#testReasoNetLoss("test.idx") +testReasoNetPred("test.idx") diff --git a/Examples/ReasoNet/testTextConvolution.py b/Examples/ReasoNet/testTextConvolution.py new file mode 100644 index 000000000000..db62f28a17e3 --- /dev/null +++ b/Examples/ReasoNet/testTextConvolution.py @@ -0,0 +1,13 @@ +import sys +import os +import numpy as np +from ReasoNet.model import text_convolution +from cntk.blocks import Placeholder, Constant +from cntk.ops import input_variable + +def testTextConvolution(): + text = np.reshape(np.arange(25.0, dtype=np.float32), (1, 5,5)) + x = input_variable(shape=(1, 5, 5,)) + c = text_convolution(3, 5, 5)(x) + v = c.eval([text])[0] + print(v) diff --git a/Examples/ReasoNet/tests/testReasoNetModel.py b/Examples/ReasoNet/tests/testReasoNetModel.py index c54d83d7d4b0..5a6046c43da0 100644 --- a/Examples/ReasoNet/tests/testReasoNetModel.py +++ b/Examples/ReasoNet/tests/testReasoNetModel.py @@ -1,7 +1,5 @@ import sys import os -import cntk.device as device -#device.set_default_device(device.cpu()) import numpy as np from ReasoNet.model import create_model, create_reader, gru_cell import ReasoNet.model as rsn @@ -12,8 +10,6 @@ from cntk import Trainer, Axis, device, combine from cntk.layers import Recurrence, Convolution from cntk.utils import _as_tuple -import cntk.ops as ops -import cntk def testModel(data): reader = create_reader(data, 10, False) @@ -34,8 +30,8 @@ def testModel(data): print(np.around(var[o], decimals=3)) def testReasoNetLoss(data): - reader = create_reader(data, 101000, False) - model = create_model(101000, 300, 3) + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) loss = rsn.loss(model) data_bind = {} for arg in loss.arguments: @@ -84,10 +80,10 @@ def testReasoNetPred(data): print('') print("Prediction: {0}\n\t=>{1}\n".format(i, np.argmax(np.reshape(i, 10)))) -def testReasoNetTrain(data, epoch_size, max_epochs=1, vocab_dim=101000, hidden_dim=300, max_rl_iter =5): - reader = create_reader(data, vocab_dim, True) - model = create_model(vocab_dim, hidden_dim, embedded_dim=100, max_rl_iter=max_rl_iter) - rsn.train(model, reader, max_epochs=max_epochs, epoch_size=epoch_size) +def testReasoNetTrain(data): + reader = create_reader(data, 10, False) + model = create_model(10, 5, 3) + rsn.train(model, reader, epoch_size=10) def testASR(data): reader = asr.create_reader(data, 10, False) @@ -121,31 +117,9 @@ def testGRU(): rt = r(a).eval({a:x}) print(rt) -def testSparse(data, vocab_dim=101000, hidden_dim=300, max_rl_iter =5): - reader = create_reader(data, vocab_dim, False) - data = reader.next_minibatch(1) - context_data = data[reader.streams.context] - context_dynamic_axes = [cntk.Axis.default_batch_axis(), cntk.Axis('Context')] - context_var= input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') - q = ops.times(1, context_var) - print(q.output.is_sparse) - su = ops.sequence.reduce_sum(ops.reshape(context_var, vocab_dim)) - o = su.eval({context_var:context_data}) - print(o) - #testGRU() #testASR("test.idx") #testModel("test.idx") #testReasoNetLoss("test.idx") -#testReasoNetLoss("test.idx") -#testReasoNetTrain("test.idx", 10, vocab_dim=10, hidden_dim=5, max_rl_iter=3) +testReasoNetTrain("test.idx") #testReasoNetPred("test.idx") - -#testReasoNetTrain("test.5.idx", 5968, max_epochs=5, vocab_dim=101000, hidden_dim=300, max_rl_iter=5) -#testReasoNetTrain("test.7.idx", 7394, max_epochs=5, vocab_dim=101000, hidden_dim=300, max_rl_iter=5) -#testReasoNetTrain("test.10.idx", 11238, max_epochs=5, vocab_dim=101000, hidden_dim=300, max_rl_iter=5) -testReasoNetTrain("test.1000.idx", 1168157, max_epochs=5, vocab_dim=101000, hidden_dim=384, max_rl_iter=5) -#testReasoNetTrain("test.1000.40k.idx", 1168157, max_epochs=5, vocab_dim=40000, hidden_dim=384, max_rl_iter=5) - - -#testSparse("test.idx", 101000) diff --git a/Examples/ReasoNet/wordvocab.py b/Examples/ReasoNet/wordvocab.py index 3bac68eeb4ae..d1ac34eb3b73 100644 --- a/Examples/ReasoNet/wordvocab.py +++ b/Examples/ReasoNet/wordvocab.py @@ -189,9 +189,9 @@ def build_bing_corpus_index(entities, words, corpus, index): index.write("{0}".format(seq_id)) for i in range(max(len(context_ids), len(query_ids))): if i < len(query_ids): - index.write(" |Q {0}:1".format(query_ids[i])) + index.write(" |Q {0}".format(query_ids[i])) if i < len(context_ids): - index.write(" |C {0}:1".format(context_ids[i])) + index.write(" |C {0}".format(context_ids[i])) index.write(" |E {0}".format(is_entity[i])) index.write(" |L {0}".format(labels[i])) index.write("\n") diff --git a/bindings/python/setup.py b/bindings/python/setup.py index e1187cfb6bc7..d771f124a249 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -42,7 +42,7 @@ CNTK_LIB_PATH = os.path.join(CNTK_PATH, "x64", "Release") else: CNTK_LIB_PATH = os.path.join( - CNTK_PATH, "build", "gpu", "release", "lib") + CNTK_PATH, "build", "gpu", "debug", "lib") print("Using CNTK sources at '%s'" % os.path.abspath(CNTK_SOURCE_PATH)) print("Using CNTK libs at '%s'" % os.path.abspath(CNTK_LIB_PATH)) From d1bed651f098ebae55befa988d9457d38895129b Mon Sep 17 00:00:00 2001 From: PengchengHe Date: Fri, 6 Jan 2017 14:52:19 +0000 Subject: [PATCH 044/120] Revert "Add training method for ReasoNet" This reverts commit 732836ab001fdde86e67e29c932603205651b4d0. --- Examples/ReasoNet/model.py | 139 ++++++------------- Examples/ReasoNet/tests/testReasoNetModel.py | 67 +-------- 2 files changed, 42 insertions(+), 164 deletions(-) diff --git a/Examples/ReasoNet/model.py b/Examples/ReasoNet/model.py index e0942299ebe0..77026b524073 100644 --- a/Examples/ReasoNet/model.py +++ b/Examples/ReasoNet/model.py @@ -22,6 +22,9 @@ # variables and stuff # ######################## +data_dir = "./data" +model_dir = "./models" + # model dimensions #vocab_dim = 40000 #embed_dim = 200 @@ -32,14 +35,6 @@ # stabilizer stabilize = Stabilizer() -def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - entities = StreamDef(field='E', shape=1, is_sparse=False), - label = StreamDef(field='L', shape=1, is_sparse=False) - )), randomize=randomize, epoch_size = size) - def text_convolution(win_size, in_dim, out_dim): #activation = _resolve_activation(activation) output_channels_shape = _as_tuple(out_dim) @@ -75,6 +70,13 @@ def text_convolution(win_size, in_dim, out_dim): apply_x = apply_x >> sigmoid return Block(apply_x, 'Convolution', Record(W=W)) +def create_reader(path, randomize, size=INFINITELY_REPEAT): + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + answer = StreamDef(field='A', shape=vocab_dim, is_sparse=True) + )), randomize=randomize, epoch_size = size) + ######################## # define the model # ######################## @@ -177,15 +179,33 @@ def attention_rlunit(context_memory, query_memory, candidate_memory, candidate_i answers = times(ans_attention, candidate_ids, name='answers') return combine([answers, termination_prob, new_status], name='ReinforcementAttention') +def attention_rlunit2(hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): + status = Placeholder(name='status') + context_memory = Placeholder(name='context_memory') + query_memory = Placeholder(name='query_memory') + candidate_memory = Placeholder(name='candidate_memory') + candidate_ids = Placeholder(name='candidate_ids') + context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim) + query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim) + context_attention = reduce_sum(element_times(context_attention_weight, context_memory), axis = 0) + query_attention = reduce_sum(element_times(query_attention_weight, query_memory), axis = 0) + attention = splice((query_attention, context_attention)) + gru = gru_cell((hidden_dim, ), name='status') + new_status = gru(attention, status).output + termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') + ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim) + answers = times(ans_attention, candidate_ids, name='ans2') + return combine([answers, termination_prob, new_status], name='ReinforcementAttention') + def seq_cross_entropy(pred, label, gama=10, name=''): pred_exp = ops.exp(pred, name='pred_exp') sum_exp = sequence.reduce_sum(pred_exp, name='sum_exp') pred_sum = element_divide(pred_exp, sequence.broadcast_as(sum_exp, pred), name='exp_divid') log_pred_sum = ops.log(pred_sum, name='log_pred') label_pred = times(label, log_pred_sum, name = 'label_softmax') - entropy = ops.negate(sequence.reduce_sum(label_pred, name='sum_log'), name=name) + entroy = ops.negate(sequence.reduce_sum(label_pred, name='sum_log'), name=name) #loss = ops.negate(sequence.reduce_sum(times(label, ops.log(pred_exp/(sequence.broadcast_as(sum_exp, pred))))), name = name) - return entropy + return entroy def mask_cross_entropy(pred, label, mask, gama=10, name=''): pred_exp = element_select(mask, ops.exp(gama*pred), 0) @@ -214,8 +234,6 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor embed_dim = hidden_dim embedding = parameter(shape=(vocab_dim, embed_dim), init=uniform(1)) - # TODO: Use Golve to initialize the embedding - # TODO: Add dropout to embedding query_embedding = times(query_sequence , embedding) context_embedding = times(context_sequence, embedding) @@ -226,6 +244,7 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor candidate_memory = sequence.scatter(sequence.gather(context_memory, candidate_filter, name='Candidate_Mem'), candidate_sc) candidate_ids = sequence.scatter(sequence.gather(candidate_indicates, candidate_filter, name = 'Candidate_Ids'), candidate_sc) entity_raw_ids = sequence.scatter(sequence.gather(reshape(context_raw, vocab_dim), candidate_filter), candidate_sc) + qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis query_memory = splice((qfwd, qbwd), name='Query_SP') # get the source (aka 'query') representation @@ -279,92 +298,12 @@ def loss(model): item_labels = sequence.reduce_sum(times(reshape(labels, (1,)), entity_ids), name='item_labels') mask = sequence.reduce_sum(entity_ids) cross_entroy = mask_cross_entropy(item_preds, item_labels, mask, name='CrossEntropyLoss') - probs = ops.element_select(mask, ops.exp(item_preds), 0, name='item_probs') - apply_loss = combine([cross_entroy, answers, labels, item_preds, probs]) - return Block(apply_loss, 'AvgSoftMaxCrossEntropy', Record(labels=item_labels)) - -#TODO: Add AUC for evaluation + return combine([cross_entroy, answers, labels, item_preds]) -def train(model, reader, max_epochs=1, save_model_flag=False, epoch_size=270000): - # Criterion nodes - criterion_loss = loss(model) - loss_func = criterion_loss.outputs[0] - eval_func = classification_error(criterion_loss.outputs[-1], criterion_loss.labels) - - # Instantiate the trainer object to drive the model training - learning_rate = 0.005 - lr_per_sample = learning_rate_schedule(learning_rate, UnitType.minibatch) - - #minibatch_size = 30000 # max(sequence_length) --> so with avg length of context=1000 this is like 30 "full samples" - minibatch_size = 5000 - - momentum_time_constant = momentum_as_time_constant_schedule(1100) - clipping_threshold_per_sample = 10.0 - gradient_clipping_with_truncation = True - learner = momentum_sgd(model.parameters, - lr_per_sample, momentum_time_constant, - gradient_clipping_threshold_per_sample=clipping_threshold_per_sample, - gradient_clipping_with_truncation=gradient_clipping_with_truncation) - trainer = Trainer(model.outputs[-1], loss_func, eval_func, learner) - - # Get minibatches of sequences to train with and perform model training - i = 0 - mbs = 0 - #epoch_size = 270000 # this number is in sequences -- need to fix (unfortunately has to be in 'elements' for now) - # for ^^, we just need to keep adding up all the samples (1 per sequence) and end the epoch once we get to 270000 - training_progress_output_freq = 1 - - # bind inputs to data from readers - data_bind = {} - label_key = None - for arg in criterion_loss.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - if arg.name == 'labels': - label_key = arg - data_bind[arg] = reader.streams.label - - for epoch in range(max_epochs): - loss_numer = 0 - metric_numer = 0 - denom = 0 - - while i < (epoch+1) * epoch_size: - - # get next minibatch of training data - #mb_train = train_reader.next_minibatch(minibatch_size_in_samples=minibatch_size, input_map=train_bind) - # TODO: When will next_minibatch ended? - # TODO: Shuffle entities? @yelong - mb_train = reader.next_minibatch(1024, input_map=data_bind) - trainer.train_minibatch(mb_train) - - # collect epoch-wide stats - samples = trainer.previous_minibatch_sample_count - loss_numer += trainer.previous_minibatch_loss_average * samples - metric_numer += trainer.previous_minibatch_evaluation_average * samples - denom += samples - - # debugging - #print("previous minibatch sample count = %d" % samples) - #print("mb_train[labels] num samples = %d" % mb_train[labels].num_samples) - #print("previous minibatch loss average = %f" % trainer.previous_minibatch_loss_average) - - if mbs % training_progress_output_freq == 0: - print("Minibatch: {}, Train Loss: {}, Train Evaluation Criterion: {}".format(mbs, - get_train_loss(trainer), get_train_eval_criterion(trainer))) - print("previous minibatch sample count = %d" % samples) - - i += mb_train[label_key].num_samples - mbs += 1 - - print("--- EPOCH %d DONE: loss = %f, errs = %f ---" % (epoch, loss_numer/denom, 100.0*(metric_numer/denom))) - - if save_model_flag: - # save the model every epoch - model_filename = os.path.join('model', "model_epoch%d.dnn" % epoch) - model.save_model(model_filename) - print("Saved model to '%s'" % model_filename) +def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + entities = StreamDef(field='E', shape=1, is_sparse=False), + label = StreamDef(field='L', shape=1, is_sparse=False) + )), randomize=randomize, epoch_size = size) diff --git a/Examples/ReasoNet/tests/testReasoNetModel.py b/Examples/ReasoNet/tests/testReasoNetModel.py index 5a6046c43da0..c8e1fff71ea2 100644 --- a/Examples/ReasoNet/tests/testReasoNetModel.py +++ b/Examples/ReasoNet/tests/testReasoNetModel.py @@ -2,7 +2,6 @@ import os import numpy as np from ReasoNet.model import create_model, create_reader, gru_cell -import ReasoNet.model as rsn import ReasoNet.asr as asr from cntk.blocks import Placeholder, Constant,initial_state_default_or_None, _is_given, _get_current_default_options from cntk.ops import input_variable, past_value, future_value @@ -22,69 +21,13 @@ def testModel(data): data_bind[arg] = reader.streams.context if arg.name == 'entities': data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(2, data_bind) + batch = reader.next_minibatch(1, data_bind) var = model.eval(batch) for o in model.outputs: print('-----------------------') print(o.name) print(np.around(var[o], decimals=3)) -def testReasoNetLoss(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - loss = rsn.loss(model) - data_bind = {} - for arg in loss.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - if arg.name == 'labels': - data_bind[arg] = reader.streams.label - batch = reader.next_minibatch(100, data_bind) - var = loss.eval(batch) - for o in loss.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - #pred = var[loss.outputs[-1]] - #for i in pred: - # print("Prediction: {0}".format(np.argmin(np.reshape(i, 10)))) - -def testReasoNetPred(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - pred = rsn.pred(model) - data_bind = {} - for arg in pred.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - if arg.name == 'labels': - data_bind[arg] = reader.streams.label - #data_bind = {model.context:reader.streams.context, model.query:reader.streams.query, model.entities:reader.streams.entities} - batch = reader.next_minibatch(100, data_bind) - var = pred.eval(batch) - for o in pred.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - pred = var[pred.outputs[-1]] - print('-----------------------') - for i in pred: - print('') - print("Prediction: {0}\n\t=>{1}\n".format(i, np.argmax(np.reshape(i, 10)))) - -def testReasoNetTrain(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - rsn.train(model, reader, epoch_size=10) - def testASR(data): reader = asr.create_reader(data, 10, False) model = asr.create_model(10, 5, 1) @@ -96,9 +39,8 @@ def testASR(data): data_bind[arg] = reader.streams.context if arg.name == 'entities': data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(2, data_bind) + batch = reader.next_minibatch(1, data_bind) var = model.eval(batch) - for o in model.outputs: print('-----------------------') print(o.name) @@ -119,7 +61,4 @@ def testGRU(): #testGRU() #testASR("test.idx") -#testModel("test.idx") -#testReasoNetLoss("test.idx") -testReasoNetTrain("test.idx") -#testReasoNetPred("test.idx") +testModel("test.idx") From 77e5d159ce473fb2646c591ca06c40076a322810 Mon Sep 17 00:00:00 2001 From: PengchengHe Date: Fri, 6 Jan 2017 14:52:22 +0000 Subject: [PATCH 045/120] Revert "Add loss/pred for ReasoNet" This reverts commit 2f94c5430bfb9a235466c29c3dd095f9acf56371. --- Examples/ReasoNet/model.py | 79 +++------------ Examples/ReasoNet/test.idx | 7 +- Examples/ReasoNet/testReasoNetModel.py | 119 ----------------------- Examples/ReasoNet/testTextConvolution.py | 13 --- 4 files changed, 16 insertions(+), 202 deletions(-) delete mode 100644 Examples/ReasoNet/testReasoNetModel.py delete mode 100644 Examples/ReasoNet/testTextConvolution.py diff --git a/Examples/ReasoNet/model.py b/Examples/ReasoNet/model.py index 77026b524073..b0eb2eee0062 100644 --- a/Examples/ReasoNet/model.py +++ b/Examples/ReasoNet/model.py @@ -10,11 +10,10 @@ from cntk.learner import momentum_sgd, momentum_as_time_constant_schedule, learning_rate_schedule, UnitType from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, tanh, times_transpose, greater, cosine_distance, element_divide, element_select -import cntk.ops as ops from cntk.blocks import LSTM, Stabilizer, _get_current_default_options, _is_given, _initializer_for, _resolve_activation, _INFERRED, Parameter, Placeholder, Block, init_default_or_glorot_uniform from cntk.layers import Recurrence, Convolution from cntk.initializer import uniform, glorot_uniform -from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input, value_to_seq +from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input from cntk.utils.debughelpers import _name_node, _node_name, _node_description, _log_node @@ -197,23 +196,6 @@ def attention_rlunit2(hidden_dim, vocab_dim, init = init_default_or_glorot_unifo answers = times(ans_attention, candidate_ids, name='ans2') return combine([answers, termination_prob, new_status], name='ReinforcementAttention') -def seq_cross_entropy(pred, label, gama=10, name=''): - pred_exp = ops.exp(pred, name='pred_exp') - sum_exp = sequence.reduce_sum(pred_exp, name='sum_exp') - pred_sum = element_divide(pred_exp, sequence.broadcast_as(sum_exp, pred), name='exp_divid') - log_pred_sum = ops.log(pred_sum, name='log_pred') - label_pred = times(label, log_pred_sum, name = 'label_softmax') - entroy = ops.negate(sequence.reduce_sum(label_pred, name='sum_log'), name=name) - #loss = ops.negate(sequence.reduce_sum(times(label, ops.log(pred_exp/(sequence.broadcast_as(sum_exp, pred))))), name = name) - return entroy - -def mask_cross_entropy(pred, label, mask, gama=10, name=''): - pred_exp = element_select(mask, ops.exp(gama*pred), 0) - label_msk = element_select(label, 1, 0) - sum_exp = ops.reduce_sum(pred_exp) - soft_max = ops.element_select(mask, ops.negate(ops.element_times(label_msk, ops.log(pred_exp/sum_exp))), 0) - return ops.reduce_sum(soft_max, name=name) - def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glorot_uniform): # Query and Doc/Context/Paragraph inputs to the model batch_axis = Axis.default_batch_axis() @@ -225,6 +207,7 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor context_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') candidate_dynamic_axes = [batch_axis, context_seq_axis] candidate_indicates = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') + candidate_filter = greater(candidate_indicates, 0) # Query sequences query_sequence = query_raw @@ -238,12 +221,10 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor context_embedding = times(context_sequence, embedding) # get source and context representations - context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis - candidate_filter = greater(candidate_indicates, 0) - candidate_sc = sequence.gather(candidate_filter, candidate_filter) - candidate_memory = sequence.scatter(sequence.gather(context_memory, candidate_filter, name='Candidate_Mem'), candidate_sc) - candidate_ids = sequence.scatter(sequence.gather(candidate_indicates, candidate_filter, name = 'Candidate_Ids'), candidate_sc) - entity_raw_ids = sequence.scatter(sequence.gather(reshape(context_raw, vocab_dim), candidate_filter), candidate_sc) + context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis + #candidate_memory = sequence.gather(context_memory, candidate_filter, name='Candidate_Mem') + candidate_memory = context_memory + candidate_ids = candidate_indicates qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis query_memory = splice((qfwd, qbwd), name='Query_SP') @@ -255,55 +236,25 @@ def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glor answers = None probs = None for i in range(0, max_rl_iter): + #arlus[i] = attention_rlu(status, context_memory, query_memory, candidate_memory, candidate_ids) arlus[i] = attention_rlu(status) status = arlus[i].outputs[2] status_controls += list(arlus[i].outputs[0:2]) if answers == None: answers = element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) + #answers = element_times(sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0]), arlus[i].outputs[0]) probs = arlus[i].outputs[1] else: answers += element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) + #answers += element_times(sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0]), arlus[i].outputs[0]) probs += arlus[i].outputs[1] final_answers = element_divide(answers, sequence.broadcast_as(probs, answers), name='final_answers') - result = combine(status_controls+[final_answers], name='ReasoNet') - return Block(result, 'ReasoNet', Record(vocab_dim=vocab_dim, hidden_dim=hidden_dim, max_ite =max_rl_iter, context=context_raw, - query=query_raw, entities=candidate_indicates, entity_masks=candidate_filter, - entity_seqs=candidate_sc, entity_ids=entity_raw_ids)) - -def pred(model): - context = model.context - entities = model.entities - wordvocab_dim = model.vocab_dim - candidate_filter = model.entity_masks - candidate_sc = model.entity_seqs - answers = sequence.scatter(model.outputs[-1], candidate_sc, name='answers_prob') - entity_ids = model.entity_ids - item_preds = sequence.reduce_sum(times(reshape(answers, (1,)), entity_ids), name = 'item_preds') - mask = sequence.reduce_sum(entity_ids, name='mask') - probs = ops.element_select(mask, ops.exp(item_preds), 0, name='item_prob') - return combine([mask, probs]) - -def loss(model): - context = model.context - entities = model.entities - wordvocab_dim = model.vocab_dim - labels_raw = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context.dynamic_axes, name='labels') - candidate_filter = model.entity_masks - candidate_sc = model.entity_seqs - answers = sequence.scatter(model.outputs[-1], candidate_sc, name='answers_prob') - entity_ids = model.entity_ids - item_preds = sequence.reduce_sum(times(reshape(answers, (1,)), entity_ids), name = 'item_preds') - labels = sequence.scatter(sequence.gather(labels_raw, candidate_filter, name='EntityLabels'), candidate_sc, name='seq_labels') - #cross_entroy = seq_cross_entroy(reshape(answers, (1,)), labels, name='CrossEntropyLoss') - item_labels = sequence.reduce_sum(times(reshape(labels, (1,)), entity_ids), name='item_labels') - mask = sequence.reduce_sum(entity_ids) - cross_entroy = mask_cross_entropy(item_preds, item_labels, mask, name='CrossEntropyLoss') - return combine([cross_entroy, answers, labels, item_preds]) + return combine(status_controls+[final_answers], name='ReasoNet') def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - entities = StreamDef(field='E', shape=1, is_sparse=False), - label = StreamDef(field='L', shape=1, is_sparse=False) + return MinibatchSource(CTFDeserializer(path, StreamDefs( + context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), + query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), + entities = StreamDef(field='E', shape=1, is_sparse=False), + label = StreamDef(field='L', shape=1, is_sparse=False) )), randomize=randomize, epoch_size = size) diff --git a/Examples/ReasoNet/test.idx b/Examples/ReasoNet/test.idx index 0316597a4c17..ac9dc92a9a34 100644 --- a/Examples/ReasoNet/test.idx +++ b/Examples/ReasoNet/test.idx @@ -1,10 +1,5 @@ 0 |Q 8:1 |C 4:1 |E 0 |L 0 -0 |Q 8:1 |C 2:1 |E 1 |L 0 +0 |Q 8:1 |C 3:1 |E 1 |L 0 0 |Q 9:1 |C 5:1 |E 0 |L 0 0 |Q 1:1 |C 6:1 |E 0 |L 0 0 |C 3:1 |E 1 |L 1 -1 |Q 2:1 |C 4:1 |E 0 |L 0 -1 |Q 1:1 |C 9:1 |E 1 |L 1 -1 |Q 7:1 |C 1:1 |E 0 |L 0 -1 |Q 1:1 |C 4:1 |E 0 |L 0 -1 |C 3:1 |E 1 |L 0 diff --git a/Examples/ReasoNet/testReasoNetModel.py b/Examples/ReasoNet/testReasoNetModel.py deleted file mode 100644 index 3513f17a6fa9..000000000000 --- a/Examples/ReasoNet/testReasoNetModel.py +++ /dev/null @@ -1,119 +0,0 @@ -import sys -import os -import numpy as np -from ReasoNet.model import create_model, create_reader, gru_cell -import ReasoNet.model as rsn -import ReasoNet.asr as asr -from cntk.blocks import Placeholder, Constant,initial_state_default_or_None, _is_given, _get_current_default_options -from cntk.ops import input_variable, past_value, future_value -from cntk.io import MinibatchSource -from cntk import Trainer, Axis, device, combine -from cntk.layers import Recurrence, Convolution -from cntk.utils import _as_tuple - -def testModel(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - data_bind = {} - for arg in model.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(2, data_bind) - var = model.eval(batch) - for o in model.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - -def testReasoNetLoss(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - loss = rsn.loss(model) - data_bind = {} - for arg in loss.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - if arg.name == 'labels': - data_bind[arg] = reader.streams.label - batch = reader.next_minibatch(100, data_bind) - var = loss.eval(batch) - for o in loss.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - #pred = var[loss.outputs[-1]] - #for i in pred: - # print("Prediction: {0}".format(np.argmin(np.reshape(i, 10)))) - -def testReasoNetPred(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - pred = rsn.pred(model) - data_bind = {} - for arg in pred.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - if arg.name == 'labels': - data_bind[arg] = reader.streams.label - #data_bind = {model.context:reader.streams.context, model.query:reader.streams.query, model.entities:reader.streams.entities} - batch = reader.next_minibatch(100, data_bind) - var = pred.eval(batch) - for o in pred.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - pred = var[pred.outputs[-1]] - print('-----------------------') - for i in pred: - print('') - print("Prediction: {0}\n\t=>{1}\n".format(i, np.argmax(np.reshape(i, 10)))) - -def testASR(data): - reader = asr.create_reader(data, 10, False) - model = asr.create_model(10, 5, 1) - data_bind = {} - for arg in model.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(2, data_bind) - var = model.eval(batch) - - for o in model.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - - -def testGRU(): - g = gru_cell(5) - x = np.reshape(np.arange(0,25, dtype=np.float32), (1,5,5)) - a = input_variable(shape=(5,), dynamic_axes=[Axis.default_batch_axis(), Axis('Seq')]) - y = np.float32([1,2,0.1,0.2,1]) - s = Constant(y) - q = g(a,s).eval({a:x}) - print(q) - r = Recurrence(gru_cell(5)) - rt = r(a).eval({a:x}) - print(rt) - -#testGRU() -#testASR("test.idx") -#testModel("test.idx") -#testReasoNetLoss("test.idx") -testReasoNetPred("test.idx") diff --git a/Examples/ReasoNet/testTextConvolution.py b/Examples/ReasoNet/testTextConvolution.py deleted file mode 100644 index db62f28a17e3..000000000000 --- a/Examples/ReasoNet/testTextConvolution.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys -import os -import numpy as np -from ReasoNet.model import text_convolution -from cntk.blocks import Placeholder, Constant -from cntk.ops import input_variable - -def testTextConvolution(): - text = np.reshape(np.arange(25.0, dtype=np.float32), (1, 5,5)) - x = input_variable(shape=(1, 5, 5,)) - c = text_convolution(3, 5, 5)(x) - v = c.eval([text])[0] - print(v) From 570a7658870ba31dd41fd937041633a2f1345aee Mon Sep 17 00:00:00 2001 From: PengchengHe Date: Fri, 6 Jan 2017 14:56:38 +0000 Subject: [PATCH 046/120] Revert "Add ReasoNet model as example" This reverts commit 09daade4f080d930af51f78508769bd589d25bc9. --- Examples/ReasoNet/__init__.py | 0 Examples/ReasoNet/__main__.py | 25 -- Examples/ReasoNet/asr.py | 256 ----------------- Examples/ReasoNet/model.py | 260 ------------------ Examples/ReasoNet/test.idx | 5 - Examples/ReasoNet/test.py | 159 ----------- Examples/ReasoNet/tests/testReasoNetModel.py | 64 ----- .../ReasoNet/tests/testTextConvolution.py | 13 - Examples/ReasoNet/wordvocab.py | 200 -------------- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 2 +- Source/CNTKv2LibraryDll/CompositeFunction.cpp | 36 +-- bindings/python/cntk/layers.py | 6 +- bindings/python/setup.py | 2 +- 13 files changed, 4 insertions(+), 1024 deletions(-) delete mode 100644 Examples/ReasoNet/__init__.py delete mode 100644 Examples/ReasoNet/__main__.py delete mode 100644 Examples/ReasoNet/asr.py delete mode 100644 Examples/ReasoNet/model.py delete mode 100644 Examples/ReasoNet/test.idx delete mode 100644 Examples/ReasoNet/test.py delete mode 100644 Examples/ReasoNet/tests/testReasoNetModel.py delete mode 100644 Examples/ReasoNet/tests/testTextConvolution.py delete mode 100644 Examples/ReasoNet/wordvocab.py diff --git a/Examples/ReasoNet/__init__.py b/Examples/ReasoNet/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/Examples/ReasoNet/__main__.py b/Examples/ReasoNet/__main__.py deleted file mode 100644 index fa1ace65e52d..000000000000 --- a/Examples/ReasoNet/__main__.py +++ /dev/null @@ -1,25 +0,0 @@ -import sys -import os -import argparse -from .wordvocab import Vocabulary - -def file_exists(src): - return (os.path.isfile(src) and os.path.exists(src)) - -parser = argparse.ArgumentParser() -parser.add_argument('-t', '--train-data', required=True, help="Path of the training data.") -parser.add_argument('-v', '--vocab', default="word_vocab.txt", help="Path of the word vocabulary data.") -parser.add_argument('-s', '--vocab-size', default=50000, type=int, help="Max size of the word vocabulary.") -parser.add_argument('-i', '--train-index', default="train.index.txt", help="The path of the featurized index file.") -args = parser.parse_args() -entity_vocab = None -word_vocab = None -if not file_exists(args.vocab): - entity_vocab, word_vocab = Vocabulary.build_bingvocab(args.train_data, args.vocab, args.vocab_size) -else: - print("Load") - entity_vocab, word_vocab = Vocabulary.load_bingvocab(args.vocab) - -if not file_exists(args.train_index): - print("Gen") - Vocabulary.build_bing_corpus_index(entity_vocab, word_vocab, args.train_data, args.train_index) diff --git a/Examples/ReasoNet/asr.py b/Examples/ReasoNet/asr.py deleted file mode 100644 index 47392a1fa51c..000000000000 --- a/Examples/ReasoNet/asr.py +++ /dev/null @@ -1,256 +0,0 @@ -""" -ReasoNet model in CNTK -@author penhe@microsoft.com -""" -import sys -import os -import numpy as np -from cntk import Trainer, Axis, device, combine -from cntk.io import MinibatchSource, CTFDeserializer, StreamDef, StreamDefs, INFINITELY_REPEAT -from cntk.learner import momentum_sgd, momentum_as_time_constant_schedule, learning_rate_schedule, UnitType -from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ - parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, tanh, times_transpose, greater, cosine_distance, element_divide, element_select -from cntk.blocks import LSTM, Stabilizer, _get_current_default_options, _is_given, _initializer_for, _resolve_activation, _INFERRED, Parameter, Placeholder, Block, init_default_or_glorot_uniform -from cntk.layers import Recurrence, Convolution -from cntk.initializer import uniform, glorot_uniform -from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input -from cntk.utils.debughelpers import _name_node, _node_name, _node_description, _log_node - - -######################## -# variables and stuff # -######################## - -data_dir = "./data" -model_dir = "./models" - -# model dimensions -#vocab_dim = 40000 -#embed_dim = 200 -#hidden_dim = 256 -#src_max_len = 120 -#ctx_max_len = 1989 - -# stabilizer -stabilize = Stabilizer() - -def text_convolution(win_size, in_dim, out_dim): - #activation = _resolve_activation(activation) - output_channels_shape = _as_tuple(out_dim) - output_rank = len(output_channels_shape) - filter_shape = (win_size, in_dim) - filter_rank = len(filter_shape) - kernel_shape = _INFERRED + filter_shape # kernel := filter plus reductionDims - - # parameters bound to this Function - init_kernel = glorot_uniform(filter_rank=filter_rank, output_rank=1) - #init_kernel = _initializer_for(init, Record(filter_rank=filter_rank, output_rank=-1)) - # BUGBUG: It is very confusing that output_rank is negative, esp. since that means count from the start. Solution: add a flag - W = Parameter(output_channels_shape + kernel_shape, init=init_kernel, name='W') # (K, C, H, W) aka [ W x H x C x K ] - #w = np.reshape(np.array([[[[2, -1, 0, -1, 2],[1,1,2,-1,-1],[1,2,0,2,1]]]], dtype = np.float32), (1, 1, 3, 5)) - #W = constant(value=w) - #b = Parameter(output_channels_shape + (1,) * len(filter_shape), init=init_bias, name='b') if bias else None # (K, 1, 1) aka [ 1 x 1 x K ] - - # expression - x = Placeholder(name='convolution_arg') - # TODO: update the parameter order of convolution() to match the optional ones as in here? (options order matches Keras) - strides = (1, 1, in_dim) - - apply_x = convolution (W, x, - strides = _as_tuple(strides), - sharing = _as_tuple(True), - auto_padding = _as_tuple(False), - lower_pad = (0, win_size/2, 0), - upper_pad = (0, (win_size-1)/2, 0) - ) -# # TODO: can we rename auto_padding to pad? - #if bias: - # apply_x = apply_x + b - apply_x = apply_x >> sigmoid - return Block(apply_x, 'Convolution', Record(W=W)) - -def create_reader(path, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - answer = StreamDef(field='A', shape=vocab_dim, is_sparse=True) - )), randomize=randomize, epoch_size = size) - - ######################## -# define the model # -######################## - -def gru_cell(shape, init=init_default_or_glorot_uniform, name=''): # (x, (h,c)) - shape = _as_tuple(shape) - - if len(shape) != 1 : - raise ValueError("gru_cell: shape must be vectors (rank-1 tensors)") - - # determine stacking dimensions - cell_shape_stacked = shape * 2 # patched dims with stack_axis duplicated 4 times - - # parameters - Wz = Parameter(cell_shape_stacked, init = init, name='Wz') - Wr = Parameter(cell_shape_stacked, init = init, name='Wr') - Wh = Parameter(cell_shape_stacked, init = init, name='Wh') - Uz = Parameter( _INFERRED + shape, init = init, name = 'Uz') - Ur = Parameter( _INFERRED + shape, init = init, name = 'Ur') - Uh = Parameter( _INFERRED + shape, init = init, name = 'Uh') - - def create_s_placeholder(): - # we pass the known dimensions here, which makes dimension inference easier - return Placeholder(shape=shape, name='S') # (h, c) - - # parameters to model function - x = Placeholder(name='gru_block_arg') - prev_status = create_s_placeholder() - - # formula of model function - Sn_1 = prev_status - - z = sigmoid(times(x, Uz, name='x*Uz') + times(Sn_1, Wz, name='Sprev*Wz'), name='z') - r = sigmoid(times(x, Ur, name='x*Ur') + times(Sn_1, Wr, name='Sprev*Wr'), name='r') - h = tanh(times(x, Uh, name='x*Uh') + times(element_times(Sn_1, r, name='Sprev*r'), Wh), name='h') - s = plus(element_times((1-z), h, name='(1-z)*h'), element_times(z, Sn_1, name='z*SPrev'), name=name) - apply_x_s = combine([s]) - apply_x_s.create_placeholder = create_s_placeholder - return apply_x_s - -def bidirectionalLSTM(hidden_dim, x, splice_outputs=True): - fwd = Recurrence(LSTM(hidden_dim), go_backwards=False) (stabilize(x)) - bwd = Recurrence(LSTM(hidden_dim), go_backwards=True ) (stabilize(x)) - if splice_outputs: - # splice the outputs together - hc = splice((fwd, bwd)) - return hc - else: - # return both (in cases where we want the 'final' hidden status) - return (fwd, bwd) - -def bidirectional_gru(hidden_dim, x, splice_outputs=True, name=''): - fwd = Recurrence(gru_cell(hidden_dim), go_backwards=False) (stabilize(x)) - bwd = Recurrence(gru_cell(hidden_dim), go_backwards=True) (stabilize(x)) - if splice_outputs: - # splice the outputs together - hc = splice((fwd, bwd), name=name) - return hc - else: - # return both (in cases where we want the 'final' hidden status) - return (fwd, bwd) - -def broadcast_as(op, seq_op, name=''): - x=placeholder_variable(shape=op.shape, name='src_op') - s=placeholder_variable(shape=seq_op.shape, name='tgt_seq') - pd = sequence.scatter(x, sequence.is_first(s, name='isf_1'), name='sct') - pout = placeholder_variable(op.shape, dynamic_axes=seq_op.dynamic_axes, name='pout') - out = element_select(sequence.is_first(s, name='isf_2'), pd, past_value(pout, name='ptv'), name='br_sel') - rlt = out.replace_placeholders({pout:sanitize_input(out), x:sanitize_input(op), s:sanitize_input(seq_op)}) - return combine([rlt], name=name) - -def cosine_similarity(src, tgt, name=''): - src_br = broadcast_as(src, tgt, name='cos_br') - sim = cosine_distance(src_br, tgt, name=name) - return sim - -def project_cosine_sim(status, memory, dim, init = init_default_or_glorot_uniform, name=''): - cell_shape = (dim, dim) - Wi = Parameter(cell_shape, init = init, name='Wi') - Wm = Parameter(cell_shape, init = init, name='Wm') - weighted_status = times(status, Wi, name = 'project_status') - weighted_memory = times(memory, Wm, name = 'project_memory') - return cosine_similarity(status, memory, name=name) - -def project_dotprod_sim(status, memory, dim, init = init_default_or_glorot_uniform, name=''): - cell_shape = (dim, dim) - Wi = Parameter(cell_shape, init = init, name='Wi') - Wm = Parameter(cell_shape, init = init, name='Wm') - weighted_status = times(status, Wi, name = 'project_status') - weighted_memory = times(memory, Wm, name = 'project_memory') - return times_transpose(broadcast_as(weighted_status, weighted_memory), weighted_memory) - -def termination_gate(status, dim, init = init_default_or_glorot_uniform, name=''): - Wt = Parameter((dim, 1), init = init, name='Wt') - return sigmoid(times(status, Wt), name=name) - -def attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids,hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): - status = Placeholder(name='status', shape=hidden_dim) - context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim, name='context_attention') - query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim, name='query_attetion') - context_attention = sequence.reduce_sum(times(context_attention_weight, context_memory), name='C-Att') - query_attention = sequence.reduce_sum(times(query_attention_weight, query_memory), name='Q-Att') - attention = splice((query_attention, context_attention), name='att-sp') - gru = gru_cell((hidden_dim, ), name='status') - new_status = gru(attention, status).output - termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') - ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim, name='ans_attention') - answers = times(ans_attention, candidate_ids, name='answers') - return combine([answers, termination_prob, new_status], name='ReinforcementAttention') - -def attention_rlunit2(hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): - status = Placeholder(name='status') - context_memory = Placeholder(name='context_memory') - query_memory = Placeholder(name='query_memory') - candidate_memory = Placeholder(name='candidate_memory') - candidate_ids = Placeholder(name='candidate_ids') - context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim) - query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim) - context_attention = reduce_sum(element_times(context_attention_weight, context_memory), axis = 0) - query_attention = reduce_sum(element_times(query_attention_weight, query_memory), axis = 0) - attention = splice((query_attention, context_attention)) - gru = gru_cell((hidden_dim, ), name='status') - new_status = gru(attention, status).output - termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') - ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim) - answers = times(ans_attention, candidate_ids, name='ans2') - return combine([answers, termination_prob, new_status], name='ReinforcementAttention') - -def set_dynamic_axes(dynamic_axes, shape): - src = placeholder_variable(shape=shape, dynamic_axes = dynamic_axes) - outp = placeholder_variable(shape=shape, dynamic_axes = dynamic_axes) - -def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glorot_uniform): - # Query and Doc/Context/Paragraph inputs to the model - batch_axis = Axis.default_batch_axis() - query_seq_axis = Axis('sourceAxis') - context_seq_axis = Axis('contextAxis') - query_dynamic_axes = [batch_axis, query_seq_axis] - query_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=query_dynamic_axes, name='query') - context_dynamic_axes = [batch_axis, context_seq_axis] - context_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') - candidate_dynamic_axes = [batch_axis, context_seq_axis] - candidate_indicates = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') - candidate_filter = greater(candidate_indicates, 0) - candidate_ids = sequence.gather(context_raw, candidate_filter) - - # Query sequences - query_sequence = query_raw - # Doc/Context sequences - context_sequence = context_raw - # embedding - embed_dim = hidden_dim - embedding = parameter(shape=(vocab_dim, embed_dim), init=uniform(1)) - - query_embedding = times(query_sequence , embedding) - context_embedding = times(context_sequence, embedding) - - # get source and context representations - context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis - candidate_memory = sequence.gather(context_memory, candidate_filter, name='Candidate_Mem') - - qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis - query_memory = splice((qfwd, qbwd), name='Query_SP') - # get the source (aka 'query') representation - status = splice((sequence.last(qfwd), sequence.first(qbwd)), name='Init_Status') # get last fwd status and first bwd status - attention_rlu = attention_rlunit(context_memory, query_memory, context_memory.output, context_sequence, hidden_dim*2, vocab_dim, init) - # TODO: the candidate_memory and candidate_ids are not in the same dynamic axes - #attention_rlu = attention_rlunit(context_memory, query_memory, candidate_memory.output, candidate_ids.output, hidden_dim*2, vocab_dim, init) - arlus = attention_rlu(status) - return combine(arlus.outputs, name='ReasoNet') - -def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - entities = StreamDef(field='E', shape=1, is_sparse=False), - label = StreamDef(field='L', shape=1, is_sparse=False) - )), randomize=randomize, epoch_size = size) diff --git a/Examples/ReasoNet/model.py b/Examples/ReasoNet/model.py deleted file mode 100644 index b0eb2eee0062..000000000000 --- a/Examples/ReasoNet/model.py +++ /dev/null @@ -1,260 +0,0 @@ -""" -ReasoNet model in CNTK -@author penhe@microsoft.com -""" -import sys -import os -import numpy as np -from cntk import Trainer, Axis, device, combine -from cntk.io import MinibatchSource, CTFDeserializer, StreamDef, StreamDefs, INFINITELY_REPEAT -from cntk.learner import momentum_sgd, momentum_as_time_constant_schedule, learning_rate_schedule, UnitType -from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ - parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, tanh, times_transpose, greater, cosine_distance, element_divide, element_select -from cntk.blocks import LSTM, Stabilizer, _get_current_default_options, _is_given, _initializer_for, _resolve_activation, _INFERRED, Parameter, Placeholder, Block, init_default_or_glorot_uniform -from cntk.layers import Recurrence, Convolution -from cntk.initializer import uniform, glorot_uniform -from cntk.utils import get_train_eval_criterion, get_train_loss, Record, _as_tuple, sanitize_input -from cntk.utils.debughelpers import _name_node, _node_name, _node_description, _log_node - - -######################## -# variables and stuff # -######################## - -data_dir = "./data" -model_dir = "./models" - -# model dimensions -#vocab_dim = 40000 -#embed_dim = 200 -#hidden_dim = 256 -#src_max_len = 120 -#ctx_max_len = 1989 - -# stabilizer -stabilize = Stabilizer() - -def text_convolution(win_size, in_dim, out_dim): - #activation = _resolve_activation(activation) - output_channels_shape = _as_tuple(out_dim) - output_rank = len(output_channels_shape) - filter_shape = (win_size, in_dim) - filter_rank = len(filter_shape) - kernel_shape = _INFERRED + filter_shape # kernel := filter plus reductionDims - - # parameters bound to this Function - init_kernel = glorot_uniform(filter_rank=filter_rank, output_rank=1) - #init_kernel = _initializer_for(init, Record(filter_rank=filter_rank, output_rank=-1)) - # BUGBUG: It is very confusing that output_rank is negative, esp. since that means count from the start. Solution: add a flag - W = Parameter(output_channels_shape + kernel_shape, init=init_kernel, name='W') # (K, C, H, W) aka [ W x H x C x K ] - #w = np.reshape(np.array([[[[2, -1, 0, -1, 2],[1,1,2,-1,-1],[1,2,0,2,1]]]], dtype = np.float32), (1, 1, 3, 5)) - #W = constant(value=w) - #b = Parameter(output_channels_shape + (1,) * len(filter_shape), init=init_bias, name='b') if bias else None # (K, 1, 1) aka [ 1 x 1 x K ] - - # expression - x = Placeholder(name='convolution_arg') - # TODO: update the parameter order of convolution() to match the optional ones as in here? (options order matches Keras) - strides = (1, 1, in_dim) - - apply_x = convolution (W, x, - strides = _as_tuple(strides), - sharing = _as_tuple(True), - auto_padding = _as_tuple(False), - lower_pad = (0, win_size/2, 0), - upper_pad = (0, (win_size-1)/2, 0) - ) -# # TODO: can we rename auto_padding to pad? - #if bias: - # apply_x = apply_x + b - apply_x = apply_x >> sigmoid - return Block(apply_x, 'Convolution', Record(W=W)) - -def create_reader(path, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - answer = StreamDef(field='A', shape=vocab_dim, is_sparse=True) - )), randomize=randomize, epoch_size = size) - - ######################## -# define the model # -######################## - -def gru_cell(shape, init=init_default_or_glorot_uniform, name=''): # (x, (h,c)) - shape = _as_tuple(shape) - - if len(shape) != 1 : - raise ValueError("gru_cell: shape must be vectors (rank-1 tensors)") - - # determine stacking dimensions - cell_shape_stacked = shape * 2 # patched dims with stack_axis duplicated 4 times - - # parameters - Wz = Parameter(cell_shape_stacked, init = init, name='Wz') - Wr = Parameter(cell_shape_stacked, init = init, name='Wr') - Wh = Parameter(cell_shape_stacked, init = init, name='Wh') - Uz = Parameter( _INFERRED + shape, init = init, name = 'Uz') - Ur = Parameter( _INFERRED + shape, init = init, name = 'Ur') - Uh = Parameter( _INFERRED + shape, init = init, name = 'Uh') - - def create_s_placeholder(): - # we pass the known dimensions here, which makes dimension inference easier - return Placeholder(shape=shape, name='S') # (h, c) - - # parameters to model function - x = Placeholder(name='gru_block_arg') - prev_status = create_s_placeholder() - - # formula of model function - Sn_1 = prev_status - - z = sigmoid(times(x, Uz, name='x*Uz') + times(Sn_1, Wz, name='Sprev*Wz'), name='z') - r = sigmoid(times(x, Ur, name='x*Ur') + times(Sn_1, Wr, name='Sprev*Wr'), name='r') - h = tanh(times(x, Uh, name='x*Uh') + times(element_times(Sn_1, r, name='Sprev*r'), Wh), name='h') - s = plus(element_times((1-z), h, name='(1-z)*h'), element_times(z, Sn_1, name='z*SPrev'), name=name) - apply_x_s = combine([s]) - apply_x_s.create_placeholder = create_s_placeholder - return apply_x_s - -def bidirectionalLSTM(hidden_dim, x, splice_outputs=True): - fwd = Recurrence(LSTM(hidden_dim), go_backwards=False) (stabilize(x)) - bwd = Recurrence(LSTM(hidden_dim), go_backwards=True ) (stabilize(x)) - if splice_outputs: - # splice the outputs together - hc = splice((fwd, bwd)) - return hc - else: - # return both (in cases where we want the 'final' hidden status) - return (fwd, bwd) - -def bidirectional_gru(hidden_dim, x, splice_outputs=True, name=''): - fwd = Recurrence(gru_cell(hidden_dim), go_backwards=False) (stabilize(x)) - bwd = Recurrence(gru_cell(hidden_dim), go_backwards=True) (stabilize(x)) - if splice_outputs: - # splice the outputs together - hc = splice((fwd, bwd), name=name) - return hc - else: - # return both (in cases where we want the 'final' hidden status) - return (fwd, bwd) - -def broadcast_as(op, seq_op, name=''): - x=placeholder_variable(shape=op.shape, name='src_op') - s=placeholder_variable(shape=seq_op.shape, name='tgt_seq') - pd = sequence.scatter(x, sequence.is_first(s, name='isf_1'), name='sct') - pout = placeholder_variable(op.shape, dynamic_axes=seq_op.dynamic_axes, name='pout') - out = element_select(sequence.is_first(s, name='isf_2'), pd, past_value(pout, name='ptv'), name='br_sel') - rlt = out.replace_placeholders({pout:sanitize_input(out), x:sanitize_input(op), s:sanitize_input(seq_op)}) - return combine([rlt], name=name) - -def cosine_similarity(src, tgt, name=''): - src_br = broadcast_as(src, tgt, name='cos_br') - sim = cosine_distance(src_br, tgt, name=name) - return sim - -def project_cosine_sim(status, memory, dim, init = init_default_or_glorot_uniform, name=''): - cell_shape = (dim, dim) - Wi = Parameter(cell_shape, init = init, name='Wi') - Wm = Parameter(cell_shape, init = init, name='Wm') - weighted_status = times(status, Wi, name = 'project_status') - weighted_memory = times(memory, Wm, name = 'project_memory') - return cosine_similarity(weighted_status, weighted_memory, name=name) - -def termination_gate(status, dim, init = init_default_or_glorot_uniform, name=''): - Wt = Parameter((dim, 1), init = init, name='Wt') - return sigmoid(times(status, Wt), name=name) - -def attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids,hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): - status = Placeholder(name='status', shape=hidden_dim) - context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim, name='context_attention') - query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim, name='query_attetion') - context_attention = sequence.reduce_sum(element_times(context_attention_weight, context_memory), name='C-Att') - query_attention = sequence.reduce_sum(element_times(query_attention_weight, query_memory), name='Q-Att') - attention = splice((query_attention, context_attention), name='att-sp') - gru = gru_cell((hidden_dim, ), name='status') - new_status = gru(attention, status).output - termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') - ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim, name='ans_attention') - answers = times(ans_attention, candidate_ids, name='answers') - return combine([answers, termination_prob, new_status], name='ReinforcementAttention') - -def attention_rlunit2(hidden_dim, vocab_dim, init = init_default_or_glorot_uniform): - status = Placeholder(name='status') - context_memory = Placeholder(name='context_memory') - query_memory = Placeholder(name='query_memory') - candidate_memory = Placeholder(name='candidate_memory') - candidate_ids = Placeholder(name='candidate_ids') - context_attention_weight = project_cosine_sim(status, context_memory, hidden_dim) - query_attention_weight = project_cosine_sim(status, query_memory, hidden_dim) - context_attention = reduce_sum(element_times(context_attention_weight, context_memory), axis = 0) - query_attention = reduce_sum(element_times(query_attention_weight, query_memory), axis = 0) - attention = splice((query_attention, context_attention)) - gru = gru_cell((hidden_dim, ), name='status') - new_status = gru(attention, status).output - termination_prob = termination_gate(new_status, dim=hidden_dim, name='prob') - ans_attention = project_cosine_sim(new_status, candidate_memory, hidden_dim) - answers = times(ans_attention, candidate_ids, name='ans2') - return combine([answers, termination_prob, new_status], name='ReinforcementAttention') - -def create_model(vocab_dim, hidden_dim, max_rl_iter=5, init=init_default_or_glorot_uniform): - # Query and Doc/Context/Paragraph inputs to the model - batch_axis = Axis.default_batch_axis() - query_seq_axis = Axis('sourceAxis') - context_seq_axis = Axis('contextAxis') - query_dynamic_axes = [batch_axis, query_seq_axis] - query_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=query_dynamic_axes, name='query') - context_dynamic_axes = [batch_axis, context_seq_axis] - context_raw = input_variable(shape=(vocab_dim), is_sparse=True, dynamic_axes=context_dynamic_axes, name='context') - candidate_dynamic_axes = [batch_axis, context_seq_axis] - candidate_indicates = input_variable(shape=(1,), is_sparse=False, dynamic_axes=context_dynamic_axes, name='entities') - candidate_filter = greater(candidate_indicates, 0) - - # Query sequences - query_sequence = query_raw - # Doc/Context sequences - context_sequence = context_raw - # embedding - embed_dim = hidden_dim - embedding = parameter(shape=(vocab_dim, embed_dim), init=uniform(1)) - - query_embedding = times(query_sequence , embedding) - context_embedding = times(context_sequence, embedding) - - # get source and context representations - context_memory = bidirectional_gru(hidden_dim, context_embedding, name='Context_Mem') # shape=(hidden_dim*2, *), *=context_seq_axis - #candidate_memory = sequence.gather(context_memory, candidate_filter, name='Candidate_Mem') - candidate_memory = context_memory - candidate_ids = candidate_indicates - - qfwd, qbwd = bidirectional_gru(hidden_dim, query_embedding, splice_outputs=False) # shape=(hidden_dim*2, *), *=query_seq_axis - query_memory = splice((qfwd, qbwd), name='Query_SP') - # get the source (aka 'query') representation - status = splice((sequence.last(qfwd), sequence.first(qbwd)), name='Init_Status') # get last fwd status and first bwd status - attention_rlu = attention_rlunit(context_memory, query_memory, candidate_memory, candidate_ids, hidden_dim*2, vocab_dim, init) - status_controls = [] - arlus = [None] * max_rl_iter - answers = None - probs = None - for i in range(0, max_rl_iter): - #arlus[i] = attention_rlu(status, context_memory, query_memory, candidate_memory, candidate_ids) - arlus[i] = attention_rlu(status) - status = arlus[i].outputs[2] - status_controls += list(arlus[i].outputs[0:2]) - if answers == None: - answers = element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) - #answers = element_times(sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0]), arlus[i].outputs[0]) - probs = arlus[i].outputs[1] - else: - answers += element_times(arlus[i].outputs[0], sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0])) - #answers += element_times(sequence.broadcast_as(arlus[i].outputs[1], arlus[i].outputs[0]), arlus[i].outputs[0]) - probs += arlus[i].outputs[1] - final_answers = element_divide(answers, sequence.broadcast_as(probs, answers), name='final_answers') - return combine(status_controls+[final_answers], name='ReasoNet') - -def create_reader(path, vocab_dim, randomize, size=INFINITELY_REPEAT): - return MinibatchSource(CTFDeserializer(path, StreamDefs( - context = StreamDef(field='C', shape=vocab_dim, is_sparse=True), - query = StreamDef(field='Q', shape=vocab_dim, is_sparse=True), - entities = StreamDef(field='E', shape=1, is_sparse=False), - label = StreamDef(field='L', shape=1, is_sparse=False) - )), randomize=randomize, epoch_size = size) diff --git a/Examples/ReasoNet/test.idx b/Examples/ReasoNet/test.idx deleted file mode 100644 index ac9dc92a9a34..000000000000 --- a/Examples/ReasoNet/test.idx +++ /dev/null @@ -1,5 +0,0 @@ -0 |Q 8:1 |C 4:1 |E 0 |L 0 -0 |Q 8:1 |C 3:1 |E 1 |L 0 -0 |Q 9:1 |C 5:1 |E 0 |L 0 -0 |Q 1:1 |C 6:1 |E 0 |L 0 -0 |C 3:1 |E 1 |L 1 diff --git a/Examples/ReasoNet/test.py b/Examples/ReasoNet/test.py deleted file mode 100644 index 33bbc562a4d4..000000000000 --- a/Examples/ReasoNet/test.py +++ /dev/null @@ -1,159 +0,0 @@ -import sys -import os -import numpy as np -from ReasoNet.model import text_convolution -from ReasoNet.model import gru_cell -from cntk.blocks import Placeholder, Constant -from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, sequence, reduce_sum, \ - parameter, times, element_times, past_value, plus, placeholder_variable, splice, reshape, constant, sigmoid, convolution, cosine_distance, times_transpose -from cntk import Axis - -def testTextConvolution(): - text = np.reshape(np.arange(25.0, dtype=np.float32), (1, 5,5)) - x = input_variable(shape=(1, 5, 5,)) - c = text_convolution(3, 5, 5)(x) - v = c.eval([text])[0] - print(v) - -def testSplice(): - a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) - b = np.reshape(np.arange(0, -25, -1, dtype=np.float32), (1,5,5)) - va = constant(value=a) - vb = constant(value=b) - vc = splice((va, vb), 2) - print(vc.shape) - print(vc.eval()) - -def cossim(a,b): - src = constant(a) - tgt = constant(b) - val = cosine_distance(src, tgt).eval() - return val - -def testCosinDistance(): - a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) - b = np.reshape(np.arange(0, 5, dtype=np.float32), (1,5)) - - src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) - tgt = input_variable(shape=(5)) - tgt_br = sequence.broadcast_as(tgt, src) - cos_seq = cosine_distance(src, tgt_br) - val = cos_seq.eval({src:[a], tgt:[b]}) - print("Cosine similarity\r\n{0}\r\n #\r\n{1}".format(a,b)) - print("==>") - print(val) - print("==================") - print("Expected: ") - for i in range(0, 5): - print("{0}:{1}".format(i, cossim(a[i], b[0]))) - -def dotproduct(a,b): - src = constant(a) - tgt = constant(b) - val = reduce_sum(element_times(src, tgt)) - return val - -def testReduceSum(): - a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) - b = np.reshape(np.arange(0, 5, dtype=np.float32), (1,5)) - - src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) - tgt = input_variable(shape=(5)) - tgt_br = sequence.broadcast_as(tgt, src) - reduceSum = reduce_sum(element_times(src, tgt_br), axis=0) - val = reduceSum.eval({src:[a], tgt:[b]}) - print("Reduce_sum\r\n{0}\r\n #\r\n{1}".format(a,b)) - print("==>") - print(val) - print("==================") - print("Expected: ") - for i in range(0, 5): - print("{0}:{1}".format(i, dotproduct(a[i], b[0]).eval())) - - -def testElementTimes(): - a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) - b = np.reshape(np.arange(0, 5, dtype=np.float32), (5)) - -# src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) -# tgt = input_variable(shape=(5)) -# tgt_br = sequence.broadcast_as(tgt, src) -# reduceSum = reduce_sum(element_times(src, tgt_br), axis=0) - val = element_times(b.reshape(5,1),a).eval() - print("ElementTimes\r\n{0}\r\n #\r\n{1}".format(a,b)) - print("==>") - print(val) - print("==================") - print("Expected: ") - for i in range(0, 5): - print("{0}:{1}".format(i, element_times(a[i], b[i]).eval())) - -def testElementTimes2(): - a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) - b = np.reshape(np.arange(0, 5, dtype=np.float32), (1, 5)) - -# src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) -# tgt = input_variable(shape=(5)) -# tgt_br = sequence.broadcast_as(tgt, src) -# reduceSum = reduce_sum(element_times(src, tgt_br), axis=0) - val = reduce_sum(element_times(b,a).eval(), axis=1).eval() - print("ElementTimes\r\n{0}\r\n #\r\n{1}".format(a,b)) - print("==>") - print(val) - print("==================") - print("Expected: ") - for i in range(0, 5): - print("{0}:{1}".format(i, dotproduct(a[i], b[0]).eval())) - -def testTimesTranspose(): - a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) - b = np.reshape(np.arange(0, 5, dtype=np.float32), (1, 5)) - -# src = input_variable(shape=(5), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) -# tgt = input_variable(shape=(5)) -# tgt_br = sequence.broadcast_as(tgt, src) -# reduceSum = reduce_sum(element_times(src, tgt_br), axis=0) - val = times_transpose(a,b).eval() - print("ElementTimes\r\n{0}\r\n #\r\n{1}".format(a,b)) - print("==>") - print(val) - print("==================") - print("Expected: ") - for i in range(0, 5): - print("{0}:{1}".format(i, dotproduct(a[i], b[0]).eval())) - -def testGRU(): - a = np.reshape(np.arange(25.0, dtype = np.float32), (5,5)) - b = np.reshape(np.arange(0, 5, dtype=np.float32), (1,5)) - src = input_variable(shape=(5, )) - #src = input_variable(shape=(1, 5, ), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) - tgt = constant(b) - gru = gru_cell(5) - o_0 = gru(src, tgt) - sgru = gru(src, o_0.output).output[0] - print(sgru.eval(a)) - - -def reduce_times_sum(first, second): - # define a recursive expression for \sum_{i=1}^t (first_i * second_i) - running_sum_ph = placeholder_variable(shape=first.shape) - print("Second: {0}".format(second.shape)) - print("Fist:{0}".format(first.shape)) - t = times(second, first) - print("Times: {0}".format(t.output.shape)) - return t - #running_sum = plus(reshape(times(second, first), shape=(5)), past_value(running_sum_ph)) - - #print("Plus: {0}".format(running_sum.output.shape)) - #running_sum.replace_placeholders({running_sum_ph : running_sum.output}) - #return sequence.last(running_sum) - -def testReduceTimesSum(): - a = np.reshape(np.float32([[0,1,0,0,0],[1,0,0,0,0],[1,0,0,0,0], [0,0,0,1,0],[0,0,0,0,1]]), (5,5)) - b = np.reshape(np.float32([0.1, 0.2, 0.2, 0.1, 0.4]), (1,5)) - src = input_variable(shape=(5, 5,)) -# src = input_variable(shape=(1, 5, ), dynamic_axes=[ Axis.default_batch_axis(), Axis("Seq")]) - tgt = constant(b) - print(reduce_times_sum(src,tgt).eval([a])) - - diff --git a/Examples/ReasoNet/tests/testReasoNetModel.py b/Examples/ReasoNet/tests/testReasoNetModel.py deleted file mode 100644 index c8e1fff71ea2..000000000000 --- a/Examples/ReasoNet/tests/testReasoNetModel.py +++ /dev/null @@ -1,64 +0,0 @@ -import sys -import os -import numpy as np -from ReasoNet.model import create_model, create_reader, gru_cell -import ReasoNet.asr as asr -from cntk.blocks import Placeholder, Constant,initial_state_default_or_None, _is_given, _get_current_default_options -from cntk.ops import input_variable, past_value, future_value -from cntk.io import MinibatchSource -from cntk import Trainer, Axis, device, combine -from cntk.layers import Recurrence, Convolution -from cntk.utils import _as_tuple - -def testModel(data): - reader = create_reader(data, 10, False) - model = create_model(10, 5, 3) - data_bind = {} - for arg in model.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(1, data_bind) - var = model.eval(batch) - for o in model.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - -def testASR(data): - reader = asr.create_reader(data, 10, False) - model = asr.create_model(10, 5, 1) - data_bind = {} - for arg in model.arguments: - if arg.name == 'query': - data_bind[arg] = reader.streams.query - if arg.name == 'context': - data_bind[arg] = reader.streams.context - if arg.name == 'entities': - data_bind[arg] = reader.streams.entities - batch = reader.next_minibatch(1, data_bind) - var = model.eval(batch) - for o in model.outputs: - print('-----------------------') - print(o.name) - print(np.around(var[o], decimals=3)) - - -def testGRU(): - g = gru_cell(5) - x = np.reshape(np.arange(0,25, dtype=np.float32), (1,5,5)) - a = input_variable(shape=(5,), dynamic_axes=[Axis.default_batch_axis(), Axis('Seq')]) - y = np.float32([1,2,0.1,0.2,1]) - s = Constant(y) - q = g(a,s).eval({a:x}) - print(q) - r = Recurrence(gru_cell(5)) - rt = r(a).eval({a:x}) - print(rt) - -#testGRU() -#testASR("test.idx") -testModel("test.idx") diff --git a/Examples/ReasoNet/tests/testTextConvolution.py b/Examples/ReasoNet/tests/testTextConvolution.py deleted file mode 100644 index db62f28a17e3..000000000000 --- a/Examples/ReasoNet/tests/testTextConvolution.py +++ /dev/null @@ -1,13 +0,0 @@ -import sys -import os -import numpy as np -from ReasoNet.model import text_convolution -from cntk.blocks import Placeholder, Constant -from cntk.ops import input_variable - -def testTextConvolution(): - text = np.reshape(np.arange(25.0, dtype=np.float32), (1, 5,5)) - x = input_variable(shape=(1, 5, 5,)) - c = text_convolution(3, 5, 5)(x) - v = c.eval([text])[0] - print(v) diff --git a/Examples/ReasoNet/wordvocab.py b/Examples/ReasoNet/wordvocab.py deleted file mode 100644 index d1ac34eb3b73..000000000000 --- a/Examples/ReasoNet/wordvocab.py +++ /dev/null @@ -1,200 +0,0 @@ -import sys -import os - -class WordFreq: - def __init__(self, word, id, freq): - self.word = word - self.id = id - self.freq = freq - -class Vocabulary: - """Build word vocabulary with frequency""" - def __init__(self, name): - self.name = name - self.size = 0 - self.__dict = {} - self.__has_index = False - - def push(self, word): - if word in self.__dict: - self.__dict[word].freq += 1 - else: - self.__dict[word] = WordFreq(word, len(self.__dict), 1) - - def build_index(self, max_size): - items = sorted(self.__dict.values(), key=lambda it : it.freq, reverse=True) - if len(items)>max_size: - del items[max_size:] - self.size=len(items) - self.__dict.clear() - for it in items: - it.id = len(self.__dict) - self.__dict[it.word] = it - self.__has_index = True - - def save(self, dst): - if not self.__has_index: - self.build_index(sys.maxsize) - if self.name != None: - dst.write("{0}\t{1}\n".format(self.name, self.size)) - for it in sorted(self.__dict.values(), key=lambda it:it.id): - dst.write("{0}\t{1}\t{2}\n".format(it.word, it.id, it.freq)) - - def load(self, src): - line = src.readline() - if line == "": - return - head = line.split() - max_size = sys.maxsize - if len(head) == 2: - self.name = head[0] - max_size = int(head[1]) - cnt = 0 - while cnt < max_size: - line = src.readline() - if line == "": - break - items = line.split() - self.__dict[items[0]] = WordFreq(items[0], int(items[1]), int(items[2])) - cnt += 1 - self.size = len(self.__dict) - self.__has_index = True - - def __getitem__(self, key): - if key in self.__dict: - return self.__dict[key] - else: - return None - - @staticmethod - def load_bingvocab(vocab_src): - """ - Load bing vocabulary from file. - - Args: - vocab_src (`str`): the file stored with the vocabulary data - Returns: - :class:`Vocabulary`: Vocabulary of the entities - :class:`Vocabulary`: Vocabulary of the words - """ - word_vocab = Vocabulary("WordVocab") - entity_vocab = Vocabulary("EntityVocab") - with open(vocab_src, 'r', encoding='utf-8') as src: - entity_vocab.load(src) - word_vocab.load(src) - return entity_vocab, word_vocab - - @staticmethod - def build_bingvocab(input_src, vocab_dst, max_size=50000): - """ - Build bing vocabulary from raw bing corpus file. - - Args: - input_src (`str`): the path of the corpus file - vocab_dst (`str`): the path of the vocabulary file to save the built vocabulary - max_size (`int`): the maxium size of the word vocabulary - Returns: - :class:`Vocabulary`: Vocabulary of the entities - :class:`Vocabulary`: Vocabulary of the words - """ - word_vocab = Vocabulary("WordVocab") - entity_vocab = Vocabulary("EntityVocab") - linenum = 0 - with open(input_src, 'r', encoding='utf-8') as src: - for line in src.readlines(): - ans, query_words, context_words = Vocabulary.parse_bing_corpus_line(line) - for q in query_words: - if q.startswith('@'): - entity_vocab.push(q) - else: - word_vocab.push(q) - for q in context_words: - if q.startswith('@'): - entity_vocab.push(q) - else: - word_vocab.push(q) - linenum += 1 - if linenum%1000==0: - print("{0} lines parsed.".format(linenum)) - entity_vocab.build_index(max_size) - word_vocab.build_index(max_size) - with open(vocab_dst, 'w', encoding='utf-8') as dst: - entity_vocab.save(dst) - word_vocab.save(dst) - return entity_vocab, word_vocab - - @staticmethod - def parse_bing_corpus_line(line): - """ - Parse bing corpus line to answer, query and context. - - Args: - line (`str`): A line of text of bing corpus - Returns: - :`str`: Answer word - :`str[]`: Array of query words - :`str[]`: Array of context/passage words - - """ - data = line.split('\t') - query = data[0] - answer = data[1] - context = data[2] - query_words = query.split() - context_words = context.split() - return answer, query_words, context_words - - @staticmethod - def build_bing_corpus_index(entities, words, corpus, index): - """ - Build featurized corpus and stored in index file in CNTKTextFormat. - - Args: - entities (class:`Vocabulary`): The entities vocabulary - words (class:`Vocabulary`): The words vocabulary - corpus (`str`): The file path of the raw corpus - index (`str`): The file path to store the featurized corpus index - """ - seq_id = 0 - with open(corpus, 'r', encoding = 'utf-8') as corp: - with open(index, 'w', encoding = 'utf-8') as index: - for line in corp.readlines(): - ans, query_words, context_words = Vocabulary.parse_bing_corpus_line(line) - ans_item = entities[ans] - query_ids = [] - context_ids = [] - is_entity = [] - labels = [] - pos = 0 - for q in context_words: - if q.startswith('@'): - item = entities[q] - context_ids += [ item.id + 1 ] - is_entity += [1] - labels += [1 if ans_item.id==item.id else 0] - else: - item = words[q] - context_ids += [ (item.id + 1) if item != None else 0 ] - is_entity += [0] - labels += [0] - pos += 1 - for q in query_words: - if q.startswith('@'): - item = entities[q] - query_ids += [ item.id + 1 ] - else: - item = words[q] - query_ids += [ (item.id + 1) if item != None else 0 ] - #Write featurized ids - index.write("{0}".format(seq_id)) - for i in range(max(len(context_ids), len(query_ids))): - if i < len(query_ids): - index.write(" |Q {0}".format(query_ids[i])) - if i < len(context_ids): - index.write(" |C {0}".format(context_ids[i])) - index.write(" |E {0}".format(is_entity[i])) - index.write(" |L {0}".format(labels[i])) - index.write("\n") - seq_id += 1 - if seq_id%1000 == 0: - print("{0} lines parsed.".format(seq_id)) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 1b8274362059..e2971b8ce287 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -364,7 +364,7 @@ namespace CNTK if (i != 0) wStrStream << L" x "; - wStrStream << ((int)m_shapeDims[i]); + wStrStream << m_shapeDims[i]; } wStrStream << L"]"; diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.cpp b/Source/CNTKv2LibraryDll/CompositeFunction.cpp index fff2a2b968bb..a97db537f569 100644 --- a/Source/CNTKv2LibraryDll/CompositeFunction.cpp +++ b/Source/CNTKv2LibraryDll/CompositeFunction.cpp @@ -339,34 +339,6 @@ namespace CNTK // when the new CNTK v2 model serialization format is ready. /*static*/ const std::wstring CompositeFunction::InternalDefaultDynamicAxisName = L"*"; /*static*/ const std::wstring CompositeFunction::InternalNoSequenceAxisName = L"__noSequenceAxis"; - - void DumpInputsTree(const Variable &var, const int &level) - { - auto owner = var.Owner(); - if(level>20) - return; - - if(owner == nullptr) - return; - auto p_inputs = owner->Inputs(); - - if(p_inputs.size() == 0) - return; - - fprintf(stderr, ">>>\n"); - for(auto iter = p_inputs.begin(); iter != p_inputs.end(); iter++) - { - fprintf(stderr, "%d|%S<=Input %S with shape %S!\n", level, ParanthesizedName(owner->Name()).c_str(), ParanthesizedName(iter->Name()).c_str(), iter->Shape().AsString().c_str()); - } - fprintf(stderr, "===\n"); - for(auto iter = p_inputs.begin(); iter != p_inputs.end(); iter++) - { - if(iter->Shape().IsUnknown()) - { - DumpInputsTree(*iter, level+1); - } - } - } // Recursively create a sub-network of ComputationNode instances corresponding to the graph of Functions // underlying the specified 'variable' and return the ComputationNode instance that corresponds to the @@ -390,13 +362,7 @@ namespace CNTK InvalidArgument("Variable%S with unknown DataType detected when compiling the Function graph!", ParanthesizedName(variable.Name()).c_str()); if (variable.Shape().IsUnknown()) - { - - fprintf(stderr, "Variable%S with unknown shape detected when compiling the Function graph, owner: %S!\n", ParanthesizedName(variable.Name()).c_str(), ParanthesizedName(variable.Owner()->Name()).c_str()); - DumpInputsTree(variable, 0); - fprintf(stderr, "Quit\n"); - InvalidArgument("Variable%S with unknown shape detected when compiling the Function graph!", ParanthesizedName(variable.Name()).c_str()); - } + InvalidArgument("Variable%S with unknown shape detected when compiling the Function graph!", ParanthesizedName(variable.Name()).c_str()); if (variable.Shape().HasInferredDimension()) InvalidArgument("Variable%S with InferredDimension for at least one axis in its shape, detected when compiling the Function graph!", ParanthesizedName(variable.Name()).c_str()); diff --git a/bindings/python/cntk/layers.py b/bindings/python/cntk/layers.py index 0eefb3804f33..6bf2882fbe6f 100644 --- a/bindings/python/cntk/layers.py +++ b/bindings/python/cntk/layers.py @@ -234,11 +234,7 @@ def previous_hook(state): f_x_h_c = over(x, prev_state) # apply the recurrent over # this returns a Function (x, (h_prev, c_prev)) -> (h, c) h_c = f_x_h_c.outputs - if type(state_forward) is tuple and len(state_forward) > 1: - replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) } - else: - replacements = {(state_forward,)[0] : h_c[0] } - + replacements = { value_forward: value for (value_forward, value) in zip(list(_as_tuple(state_forward)), h_c) } f_x_h_c.replace_placeholders(replacements) # resolves state_forward := h_c h = f_x_h_c.outputs[0] # 'h' is a Variable (the output of a Function that computed it) if _trace_layers: diff --git a/bindings/python/setup.py b/bindings/python/setup.py index d771f124a249..e1187cfb6bc7 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -42,7 +42,7 @@ CNTK_LIB_PATH = os.path.join(CNTK_PATH, "x64", "Release") else: CNTK_LIB_PATH = os.path.join( - CNTK_PATH, "build", "gpu", "debug", "lib") + CNTK_PATH, "build", "gpu", "release", "lib") print("Using CNTK sources at '%s'" % os.path.abspath(CNTK_SOURCE_PATH)) print("Using CNTK libs at '%s'" % os.path.abspath(CNTK_LIB_PATH)) From 71c5aaba063ab82c745bcc4288a93ce1ea0685f5 Mon Sep 17 00:00:00 2001 From: Junjie Qian Date: Tue, 3 Jan 2017 12:29:58 -0800 Subject: [PATCH 047/120] MPI_info: Path check and error handling for MPI exec MPI_check: Check and selection of the installed MPI MPI_info: default unknown fix for MPI name and version --- Makefile | 2 +- Source/CNTK/prebuild.bat | 20 +++++++++++++++-- Tools/generate_build_info | 22 ++++++++++-------- configure | 47 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 9914d52bdce4..595cc027ebb8 100644 --- a/Makefile +++ b/Makefile @@ -75,7 +75,7 @@ endif # The mpic++ wrapper only adds MPI specific flags to the g++ command line. # The actual compiler/linker flags added can be viewed by running 'mpic++ --showme:compile' and 'mpic++ --showme:link' -CXX = mpic++ +CXX = $(MPI_PATH)/bin/mpic++ SSE_FLAGS = -msse4.1 -mssse3 PROTOC = $(PROTOBUF_PATH)/bin/protoc diff --git a/Source/CNTK/prebuild.bat b/Source/CNTK/prebuild.bat index dcdb39f09524..5a58e163dc45 100644 --- a/Source/CNTK/prebuild.bat +++ b/Source/CNTK/prebuild.bat @@ -98,8 +98,24 @@ if not %l_build_target% == CPU-only ( ) ) -echo #define _MPI_NAME_ "msmpi" >> buildinfo.h$$ -for /f "tokens=6 delims=] " %%i in ('mpiexec -? ^| findstr Version') do echo #define _MPI_VERSION_ "%%i" >> buildinfo.h$$ +:: MPI info +set MPI_NAME="Unknown" +set MPI_VERSION="Unknown" +where -q mpiexec && mpiexec.exe -help > NUL 2>&1 +if not errorlevel 1 ( + for /f "tokens=1 delims= " %%i in ('mpiexec -help ^| findstr Version') do ( + if "%%i" == "Microsoft" ( + set MPI_NAME="Microsoft MPI" + for /f "tokens=6 delims=] " %%i in ('mpiexec -help ^| findstr Version') do set MPI_VERSION="%%i" + ) else if "%%i" == "Intel" ( + set MPI_NAME="Intel MPI" + for /f "tokens=8 delims= " %%i in ('mpiexec -help ^| findstr Version') do set MPI_VERSION="%%i" + ) + ) +) +echo #define _MPI_NAME_ %MPI_NAME% >> buildinfo.h$$ +echo #define _MPI_VERSION_ %MPI_VERSION% >> buildinfo.h$$ + echo #endif >> buildinfo.h$$ ::: update file only if it changed (otherwise CNTK.cpp will get rebuilt each time) diff --git a/Tools/generate_build_info b/Tools/generate_build_info index 036e2b7f9e29..3def1dc0a674 100755 --- a/Tools/generate_build_info +++ b/Tools/generate_build_info @@ -20,6 +20,8 @@ # BUILDER (user under which the build was done) # BUILDMACHINE (build machine) # BUILDPATH (build path) +# MPI_NAME (mpi distribution) +# MPI_VERSION (mpi version) usage () { @@ -142,15 +144,17 @@ if [ "$MATHLIB" = "mkl" -a "$MKL_THREADING" = "sequential" ]; then fi # MPI info -which ompi_info > /dev/null && -{ - MPI_NAME=openmpi - MPI_VERSION=`ompi_info --parsable | grep ident | cut -f 2 -d ":"` -} || -{ - MPI_NAME=mvapich2 - MPI_VERSION=`mpiname -v` -} +MPI_NAME=Unknown +MPI_VERSION=Unknown +if hash ${MPI_PATH}/bin/mpirun 2>/dev/null; then + if [ -e ${MPI_PATH}/bin/ompi_info ]; then + MPI_NAME="Open MPI" + MPI_VERSION=`${MPI_PATH}/bin/ompi_info --parsable | grep ident | cut -f 2 -d ":"` + elif [ -e ${MPI_PATH}/bin/mpirun_rsh ]; then + MPI_NAME=`${MPI_PATH}/bin/mpiname` + MPI_VERSION=`${MPI_PATH}/bin/mpiname -v` + fi +fi # Build machine info BUILDER=$USER diff --git a/configure b/configure index 60afd72ef658..25f6d4eab533 100755 --- a/configure +++ b/configure @@ -41,6 +41,10 @@ boost_check=include/boost/test/unit_test.hpp protobuf_path= protobuf_check=lib/libprotobuf.a +# MPI library +mpi_path= +mpi_check=include/mpi.h + have_kaldi=no kaldi_path= kaldi_check=src/kaldi.mk @@ -113,6 +117,7 @@ default_opencvs="opencv-3.1.0 opencv-3.0.0" default_protobuf="protobuf-3.1.0" default_libzips="libzip-1.1.2" default_swig="swig-3.0.10" +default_mpi="mpi" function default_paths () { @@ -215,6 +220,11 @@ function find_libzip () find_dir "$default_libzips" "$libzip_check" } +function find_mpi () +{ + find_dir "$default_mpi" "$mpi_check" +} + function is_hardlinked () { r=no @@ -348,6 +358,7 @@ function show_help () echo " --with-py34-path[=directory] $(show_default $(find_python 34))" echo " --with-py35-path[=directory] $(show_default $(find_python 35))" echo " --with-swig[=directory] $(show_default $(find_swig))" + echo " --with-mpi[=directory] $(show_default $(find_mpi))" echo "Libraries search path:" for head in $(default_paths) @@ -802,6 +813,25 @@ do fi fi ;; + --with-mpi*) + if test x$optarg = x + then + mpi_path=$(find_mpi) + if test x$mpi_path = x + then + echo "Cannot find mpi directory." + exit 1 + fi + else + if test $(check_dir $optarg $mpi_check) = yes + then + mpi_path=$optarg + else + echo "Invalid mpi directory $optarg" + exit 1 + fi + fi + ;; *) echo Invalid option $key show_help @@ -1004,6 +1034,20 @@ then fi fi +if test x$mpi_path = x +then + mpi_path=$(find_mpi) + if test x${mpi_path} = x + then + echo Cannot locate MPI library. See + echo https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Linux#open-mpi + echo for installation instructions. + exit 1 + else + echo Found MPI at $mpi_path + fi +fi + config=$build_top/Config.make echo Generating $config echo "#Configuration file for cntk" > $config @@ -1059,6 +1103,9 @@ fi if test x$protobuf_path != x; then echo PROTOBUF_PATH=$protobuf_path >> $config fi +if test x$mpi_path != x; then + echo MPI_PATH=$mpi_path >> $config +fi if test $enable_asgd = yes ; then echo CNTK_ENABLE_ASGD=true >> $config From e25a4c4f5096ed5bcd432293055ad5e18e9fddbb Mon Sep 17 00:00:00 2001 From: jplu Date: Tue, 29 Mar 2016 16:16:50 +0200 Subject: [PATCH 048/120] Change the mpic++ binary path --- Makefile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Makefile b/Makefile index 595cc027ebb8..4eee8558de3a 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,8 @@ # PYTHON27_PATH= path to Python 2.7 interpreter # PYTHON34_PATH= path to Python 3.4 interpreter # PYTHON35_PATH= path to Python 3.5 interpreter +# MPI_PATH= path to MPI installation, so $(MPI_PATH) exists +# defaults to /usr/local/mpi # These can be overridden on the command line, e.g. make BUILDTYPE=debug # TODO: Build static libraries for common dependencies that are shared by multiple From 7af3a7c0e46cb12f873f1289400a9c5d86746662 Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Fri, 6 Jan 2017 16:41:11 -0800 Subject: [PATCH 049/120] incorporated CR feedback --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 10 +------- Source/CNTKv2LibraryDll/BackCompat.cpp | 2 +- Source/CNTKv2LibraryDll/CompositeFunction.cpp | 7 ++---- Source/CNTKv2LibraryDll/Function.cpp | 4 ++-- Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 15 ++++++------ Source/CNTKv2LibraryDll/PrimitiveFunction.h | 1 - .../ConvolutionalNodes.h | 6 ++++- bindings/python/cntk/ops/__init__.py | 3 +-- bindings/python/cntk/ops/tests/kernel_test.py | 23 ++++++++++--------- 9 files changed, 32 insertions(+), 39 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index de88d2e70174..b0e098cfa48f 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -2954,20 +2954,12 @@ namespace CNTK const NDShape& upperPad = {0}, const std::wstring& name = L""); - /// - /// Unpooling types - /// - enum class UnpoolingType - { - Max - }; - /// /// Create an instance of the CNTK built-in Unpooling operation on specified tensor input operands with the specified type and shape /// CNTK_API FunctionPtr Unpooling(const Variable& operand, const Variable& poolingInput, - UnpoolingType UnpoolingType, + PoolingType UnpoolingType, const NDShape& UnpoolingWindowShape, const NDShape& strides = { 1 }, const std::vector& autoPadding = { false }, diff --git a/Source/CNTKv2LibraryDll/BackCompat.cpp b/Source/CNTKv2LibraryDll/BackCompat.cpp index 52541bda0502..35a793ede3d7 100644 --- a/Source/CNTKv2LibraryDll/BackCompat.cpp +++ b/Source/CNTKv2LibraryDll/BackCompat.cpp @@ -347,7 +347,7 @@ namespace CNTK else if (node->OperationName() == OperationNameOf(MaxUnpoolingNode)) { auto unpoolingNode = node->As>(); - primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameUnpoolingType] = (size_t)UnpoolingType::Max; + primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNamePoolingType] = (size_t)PoolingType::Max; primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameUnpoolingWindowShape] = AsNDShape(unpoolingNode->KernelShape()); primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameStrides] = AsNDShape(unpoolingNode->Strides()); primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameAutoPadding] = AsDictionaryValueVector(unpoolingNode->AutoPad()); diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.cpp b/Source/CNTKv2LibraryDll/CompositeFunction.cpp index 40e42306d9ea..f26c3bbd8d5e 100644 --- a/Source/CNTKv2LibraryDll/CompositeFunction.cpp +++ b/Source/CNTKv2LibraryDll/CompositeFunction.cpp @@ -579,16 +579,13 @@ namespace CNTK } case PrimitiveOpType::Unpooling: { - UnpoolingType unpoolingType = (UnpoolingType)(functionConfig[PrimitiveFunction::AttributeNameUnpoolingType].Value()); auto unpoolingWindowShape = functionConfig[PrimitiveFunction::AttributeNameUnpoolingWindowShape].Value(); auto strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); auto lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); auto upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); - if (unpoolingType == UnpoolingType::Max) - computationNodePtr = New>(network->GetDeviceId(), internalNodeName, AsTensorShape(unpoolingWindowShape), AsTensorShape(strides), autoPadding, AsTensorShape(lowerPad), AsTensorShape(upperPad), ImageLayoutKind::CHW); - else - LogicError("Unknown unpooling type"); + //We only get here after validation so it is safe to assume unpooling is max + computationNodePtr = New>(network->GetDeviceId(), internalNodeName, AsTensorShape(unpoolingWindowShape), AsTensorShape(strides), autoPadding, AsTensorShape(lowerPad), AsTensorShape(upperPad), ImageLayoutKind::CHW); break; } case PrimitiveOpType::SumAll: diff --git a/Source/CNTKv2LibraryDll/Function.cpp b/Source/CNTKv2LibraryDll/Function.cpp index 58e90c54608a..a01715f20ee9 100644 --- a/Source/CNTKv2LibraryDll/Function.cpp +++ b/Source/CNTKv2LibraryDll/Function.cpp @@ -1067,7 +1067,7 @@ namespace CNTK FunctionPtr Unpooling(const Variable& operand, const Variable& poolingInput, - UnpoolingType unpoolingType, + PoolingType unpoolingType, const NDShape& poolingWindowShape, const NDShape& strides, const std::vector& autoPadding, @@ -1076,7 +1076,7 @@ namespace CNTK const std::wstring& name) { auto additionalProperties = Dictionary(); - additionalProperties[PrimitiveFunction::AttributeNameUnpoolingType] = (size_t)unpoolingType; + additionalProperties[PrimitiveFunction::AttributeNamePoolingType] = (size_t)unpoolingType; additionalProperties[PrimitiveFunction::AttributeNameUnpoolingWindowShape] = poolingWindowShape; additionalProperties[PrimitiveFunction::AttributeNameStrides] = strides; additionalProperties[PrimitiveFunction::AttributeNameAutoPadding] = AsDictionaryValueVector(autoPadding); diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index 805610402a37..66ed00c21fd6 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -71,7 +71,6 @@ namespace CNTK /*static*/ const std::wstring PrimitiveFunction::AttributeNameRecurrentOp = L"recurrentOp"; /*static*/ const std::wstring PrimitiveFunction::AttributeNameRngSeed = L"rngSeed"; /*static*/ const std::wstring PrimitiveFunction::AttributeNameRngOffset = L"rngOffset"; - /*static*/ const std::wstring PrimitiveFunction::AttributeNameUnpoolingType = L"unpoolingType"; /*static*/ const std::wstring PrimitiveFunction::AttributeNameUnpoolingWindowShape = L"unpoolingWindowShape"; /*static*/ std::vector PrimitiveFunction::GetOutputVariables(PrimitiveOpType op, @@ -352,11 +351,13 @@ namespace CNTK assert(inputs.size() == 2); auto inputShape = inputs[0].Shape(); - auto poolInputShape = inputs[1].Shape(); - outputShape = poolInputShape; + outputShape = inputs[1].Shape(); + PoolingType unpoolingType = (PoolingType)(functionConfig[PrimitiveFunction::AttributeNamePoolingType].Value()); + if (unpoolingType != PoolingType::Max) + LogicError("Only max unpooling is currently supported"); - // Finding the shape of an unpooling operation is ambiguous - // For example a 4x4x1 input with a 5x5 kernel a stride of 2x2 + // Finding the shape of an unpooling operation from the input to be unpooled alone is ambiguous + // For example a 4x4 input with a 5x5 kernel a stride of 2x2 // and padding could have resulted from pooling a 7x7 or 8x8 image // Therefore what needs to happen here is to check whether the // outputShape can be pooled into the inputShape using the specified attributes @@ -370,7 +371,7 @@ namespace CNTK NDShape inferredInputShape = ConvolutionOpOutputShape(PrimitiveOpType::Pooling, outputShape, unpoolingWindowShape, inputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, false, inferDimensions); if (inferredInputShape != inputShape) - RuntimeError("The shape of the unpooling operand is different from the result of pooling the poolingInput argument using the provided options"); + RuntimeError("The shape of the unpooling operand %ls is different from the result of pooling the poolingInput argument using the provided options %ls", inputShape.AsString(), inferredInputShape.AsString()); break; } @@ -668,7 +669,7 @@ namespace CNTK dict[inputsKey] = std::move(inputUids); - if (m_op == PrimitiveOpType::Block) + if (m_op == PrimitiveOpType::Unpooling) { auto blockCompositeFunc = dynamic_cast(BlockComposite().get()); dict[blockFunctionCompositeKey] = blockCompositeFunc->SerializeBlockComposite(); diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.h b/Source/CNTKv2LibraryDll/PrimitiveFunction.h index 29766e3e5e2c..4067052738f0 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.h +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.h @@ -219,7 +219,6 @@ namespace CNTK static const std::wstring AttributeNameNumLayers; static const std::wstring AttributeNameHiddenSize; static const std::wstring AttributeNameRecurrentOp; - static const std::wstring AttributeNameUnpoolingType; static const std::wstring AttributeNameUnpoolingWindowShape; protected: diff --git a/Source/ComputationNetworkLib/ConvolutionalNodes.h b/Source/ComputationNetworkLib/ConvolutionalNodes.h index 6b2e1a3af611..6a34ea6c75a9 100644 --- a/Source/ComputationNetworkLib/ConvolutionalNodes.h +++ b/Source/ComputationNetworkLib/ConvolutionalNodes.h @@ -919,7 +919,11 @@ class MaxUnpoolingNode : public ConvolutionNodeBase, public NumInputs< auto inferredShape = ConvolveGeometry::ComputeOutputShape(outputShape, m_kernelShape, m_mapCount, m_stride, m_sharing, m_autoPad, m_lowerPad, m_upperPad); if (inputShape != inferredShape) - InvalidArgument("The shape of the unpooling operand is different from the result of pooling the poolingInput argument using the provided options"); + InvalidArgument("%ls %ls the shape of the unpooling operand %ls is different from " + "the result of pooling the poolingInput argument using" + "the provided options %ls", NodeName().c_str(), OperationName().c_str(), + static_cast(inputShape).c_str(), + static_cast(inferredShape).c_str()); SetDims(outputShape, HasMBLayout()); if (isFinalValidationPass) diff --git a/bindings/python/cntk/ops/__init__.py b/bindings/python/cntk/ops/__init__.py index 32f148854efa..d0e9c4ba0890 100644 --- a/bindings/python/cntk/ops/__init__.py +++ b/bindings/python/cntk/ops/__init__.py @@ -433,8 +433,7 @@ def pooling(operand, pooling_type, pooling_window_shape, strides=(1,), auto_padd lower_pad, upper_pad, name) -from cntk.cntk_py import UnpoolingType_Max -MAX_UNPOOLING = UnpoolingType_Max +MAX_UNPOOLING = PoolingType_Max @typemap def unpooling(operand, pooling_input, unpooling_type, unpooling_window_shape, strides=(1,), auto_padding=[False], lower_pad=(0,), upper_pad=(0,), name=''): diff --git a/bindings/python/cntk/ops/tests/kernel_test.py b/bindings/python/cntk/ops/tests/kernel_test.py index 32f7eaac6fe6..f6daecb7d1bc 100644 --- a/bindings/python/cntk/ops/tests/kernel_test.py +++ b/bindings/python/cntk/ops/tests/kernel_test.py @@ -74,13 +74,13 @@ def test_op_convolution_without_padding(convolution_map, convolution_input, devi ([1, 1, 1, 3, 3], # input_size [1, 2, 2], # convolution size [[[[ 19, 25, 10], - [ 37, 43, 16], + [ 37, 43, 16], [ 7, 8, 0]]]]) # result ] # this test handles convolution with asymmetric padding, in particular, with auto_padding is set to True -# and the kernel shape is even +# and the kernel shape is even @pytest.mark.parametrize("input_size, conv_size, result", ASYM_CONVOLUTION_DATA) -def test_asym_convolution(input_size, conv_size, result, device_id, precision): +def test_asym_convolution(input_size, conv_size, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] dev = cntk_device(device_id) @@ -95,9 +95,9 @@ def test_asym_convolution(input_size, conv_size, result, device_id, precision): needs_gradient=False, name='a') - # do the same for convolution kernel + # do the same for convolution kernel total_size = np.prod(conv_size) - y = np.arange(total_size, dtype=dt) + y = np.arange(total_size, dtype=dt) conv_map = constant(value=y.reshape(conv_size), device=dev) from cntk import convolution @@ -114,7 +114,7 @@ def test_asym_convolution(input_size, conv_size, result, device_id, precision): ([1, 1, 1, 6, 6], # input_size (1, 5, 5), # pooling_window (1, 3, 3), # strides - [True], # padding flag + [True], # padding flag [[[[ 21, 23], [ 33, 35]]]]), # result ([1, 1, 1, 8, 8], @@ -123,10 +123,10 @@ def test_asym_convolution(input_size, conv_size, result, device_id, precision): [False], [[[[ 27 ]]]]) ] -# the pooling geometry test also tests convolution geometry since they go through the same path -# in the CPU code +# the pooling geometry test also tests convolution geometry since they go through the same path +# in the CPU code @pytest.mark.parametrize("input_size, pooling_window, strides, padding, result", POOLING_GEOMETRY_DATA) -def test_op_pooling_geometry(input_size, pooling_window, strides, padding, result, device_id, precision): +def test_op_pooling_geometry(input_size, pooling_window, strides, padding, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] # fill input operand with a sequence 1,2,3,... til total size and then @@ -264,6 +264,7 @@ def test_op_max_pooling(input_size, pooling_window, strides, autopad, result, de def test_op_max_unpooling(input_size, pooling_window, strides, autopad, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] + # fill input operand with a sequence 1,2,3,... til total size and then # resize to input_size total_size = np.prod(input_size) @@ -329,7 +330,7 @@ def test_op_roipooling(input_map, input_rois, expected_fwd, expected_bkwd, devic roi_input = AA(input_rois, dtype=dt) exp_fwd_value = AA(expected_fwd, dtype=dt) exp_bkwd_value = AA(expected_bkwd, dtype=dt) - + # adding batch, sequence and roi axis exp_fwd_value.shape = (1,1,1) + exp_fwd_value.shape exp_bkwd_value.shape = (1,1) + exp_bkwd_value.shape @@ -348,7 +349,7 @@ def test_op_roipooling(input_map, input_rois, expected_fwd, expected_bkwd, devic # adding batch and sequence axis conv_input.shape = (1,1) + conv_input.shape roi_input.shape = (1,1) + roi_input.shape - + from cntk import roipooling input_op = roipooling(a, b, (3,3)) From cddb1b9dd857fdf8c392b563f8ff7aff131dda2d Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Fri, 6 Jan 2017 16:48:03 -0800 Subject: [PATCH 050/120] leftover convolution change --- bindings/python/cntk/ops/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bindings/python/cntk/ops/__init__.py b/bindings/python/cntk/ops/__init__.py index d0e9c4ba0890..e0fc910a8705 100644 --- a/bindings/python/cntk/ops/__init__.py +++ b/bindings/python/cntk/ops/__init__.py @@ -357,8 +357,11 @@ def convolution(convolution_map, operand, strides=(1,), sharing=[True], ''' from cntk.cntk_py import convolution operand = sanitize_input(operand) - return convolution(convolution_map, operand, tuple(strides), sharing, auto_padding, - tuple(lower_pad), tuple(upper_pad), transpose, + strides = sanitize_shape(strides) + lower_pad = sanitize_shape(lower_pad) + upper_pad = sanitize_shape(upper_pad) + return convolution(convolution_map, operand, strides, sharing, auto_padding, + lower_pad, upper_pad, transpose, max_temp_mem_size_in_samples, name) From b6c0b19f1920abe0ce17cefbbd545ad4802252c1 Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Fri, 6 Jan 2017 18:28:04 -0800 Subject: [PATCH 051/120] CNTK v2 library: Allow setting Function instance names post initial construction --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 55 +++++++++++-------- .../API/CNTKLibraryInternals.h | 3 + Source/CNTKv2LibraryDll/Common.cpp | 11 ++++ Source/CNTKv2LibraryDll/Function.cpp | 8 +++ bindings/csharp/Swig/cntk_cs.i | 2 + bindings/python/cntk/cntk_py.i | 1 + bindings/python/cntk/ops/functions.py | 12 ++++ .../python/cntk/ops/tests/function_tests.py | 18 ++++++ 8 files changed, 86 insertions(+), 24 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index e2971b8ce287..7024620e7b9e 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -1512,7 +1512,7 @@ namespace CNTK CNTK_API Variable(const FunctionPtr& function); /// - /// Implicit conversion to a FunctionPtr; creates a pass through primitive function + /// Implicit conversion to a FunctionPtr; creates a pass through primitive Function /// CNTK_API operator FunctionPtr() const; @@ -2347,7 +2347,7 @@ namespace CNTK /// /// Represents a function (optionally differentiable w.r.t. its inputs) /// A Function denotes a symbolic computation with zero or more input arguments and one or more outputs. - /// A Function may be primitive or composite (comprised of other function instances whose inputs and outputs are wired together). + /// A Function may be primitive or composite (comprised of other Function instances whose inputs and outputs are wired together). /// A Function effectively is a computation graph composed of other primitive Functions (denoting computation) as nodes and Variable objects /// (denoting data) as the edges and leaves of the graph. /// Function class inherits from IDictionarySerializable to allow derived 'Function' types to specify custom serialization procedure. @@ -2361,17 +2361,17 @@ namespace CNTK public: /// /// Computes and stores the values of specified variables in the 'outputs' map, using provided 'inputs' values corresponding - /// to each leaf variable of the function of VariableKind 'Input'. + /// to each leaf variable of the Function of VariableKind 'Input'. /// The variables specified in the 'outputs' map denote the subset of 'this' Function's output variables that the caller wants to obtain values of. /// Callers may specify the storage to be used for storing the 'outputs' Values or pass null in which case the implementation allocates the actual storage /// for the 'outputs' for which the ValuePtr mapping was left null by the caller. /// The optional 'outputsToRetainBackwardStateFor' parameter specifies the subset of the Function's output variables for which gradients will be specified /// in a subsequent Backward call for backpropagation. /// The method returns a BackPropState object containing all intermediate variable values needed during backpropagation of gradients from the - /// 'outputsToRetainBackwardStateFor' outputs of the function to any of the inputs of the Function, in a subsequent Backward call. + /// 'outputsToRetainBackwardStateFor' outputs of the Function to any of the inputs of the Function, in a subsequent Backward call. /// Note that the returned BackPropState instance also stores a reference to the supplied 'inputs' Values and generated 'outputs' Values /// and the user is responsible for ensuring that the contents of the inputs and outputs are unchanged until after any uses of the BackPropState instance - /// for backpropagating gradients through this function. + /// for backpropagating gradients through this Function. /// virtual BackPropStatePtr Forward(const std::unordered_map& arguments, std::unordered_map& outputs, @@ -2426,25 +2426,32 @@ namespace CNTK /// /// Deserializes a Function from the dictionary. - /// TODO: add a second overload with a 'function builder' parameter that would allow hooking + /// TODO: add a second overload with a 'Function builder' parameter that would allow hooking /// user-defined op-codes with custom functionality. /// CNTK_API static FunctionPtr Deserialize(const Dictionary& dictionary, const ::CNTK::DeviceDescriptor& device = DeviceDescriptor::UseDefaultDevice()); public: /// - /// Returns the name of 'this' function. + /// Returns the name of 'this' Function. /// const std::wstring& Name() const { return m_name; } /// - /// Returns the internally generated unique name of the function + /// Sets the name of 'this' Function. + /// Setting the name of a Function is only allowed if the Function does not already have a name. + /// Calling this method, when 'this' Function already has a name, results in an exception. + /// + CNTK_API void SetName(const std::wstring& name); + + /// + /// Returns the internally generated unique name of the Function /// const std::wstring& Uid() const { return m_uid; } /// /// Returns the primitive Function at the root of the graph of Functions underlying this Function. - /// If 'this' Function itself is a primitive function then (this->RootFunction() == this). + /// If 'this' Function itself is a primitive Function then (this->RootFunction() == this). /// FunctionPtr RootFunction() const { @@ -2452,30 +2459,30 @@ namespace CNTK } /// - /// Returns a boolean indicating if this Function is a composite function + /// Returns a boolean indicating if this Function is a composite Function /// bool IsComposite() const { return (m_rootFunction != nullptr); } /// - /// Returns a boolean indicating if this Function is a primitive function + /// Returns a boolean indicating if this Function is a primitive Function /// bool IsPrimitive() const { return !IsComposite(); } /// - /// Returns a boolean indicating if this Function is a block function which is basically + /// Returns a boolean indicating if this Function is a block Function which is basically /// a composite encapsulated as an opaque block which appears as a primitive during traversing /// the graph of Functions that this block is part of. /// CNTK_API bool IsBlock() const; /// - /// Returns the composite function underlying this block Function. + /// Returns the composite Function underlying this block Function. /// Throws an exception of this is not a block Function /// CNTK_API FunctionPtr BlockComposite() const; /// - /// Returns the mapping from the arguments of the composite underlying this block function + /// Returns the mapping from the arguments of the composite underlying this block Function /// to the Variables that they are bound to in the outer graph of Functions that this /// block Function is part of. /// @@ -2565,7 +2572,7 @@ namespace CNTK CNTK_API FunctionPtr ReplacePlaceholder(const Variable& placeholderReplacement); /// - /// Save this function graph into a model file. + /// Save this Function graph into a model file. /// CNTK_API void SaveModel(const std::wstring& modelFile); @@ -2575,12 +2582,12 @@ namespace CNTK CNTK_API void RestoreModel(const std::wstring& modelFilePath); /// - /// Load a function from a model file + /// Load a Function from a model file /// CNTK_API static FunctionPtr LoadModel(const std::wstring& modelFile, const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice()); /// - /// Prints the entire graph underlying this function to stderr + /// Prints the entire graph underlying this Function to stderr /// CNTK_API void PrintGraph() const; @@ -2590,8 +2597,8 @@ namespace CNTK /// CNTK_API Function(const std::vector& inputs, const std::vector& outputs, Dictionary&& functionConfig, const std::wstring& name = L"", const std::wstring& uid = Internal::GenerateUid(L"UserDefinedFunction")); - /// Restores the state of the 'this' function in place using the provided dictionary. - /// Structurally, 'this' function graph has to be identical to the state captured in the dictionary. + /// Restores the state of the 'this' Function in place using the provided dictionary. + /// Structurally, 'this' Function graph has to be identical to the state captured in the dictionary. CNTK_API virtual void RestoreFromCheckpoint(const Dictionary& dictionary); /// @@ -2654,7 +2661,7 @@ namespace CNTK std::vector m_inputs; std::vector m_outputs; - FunctionPtr m_rootFunction; // nullptr for primitive function instances + FunctionPtr m_rootFunction; // nullptr for primitive Function instances std::wstring m_name; std::wstring m_uid; Dictionary m_attributes; @@ -3134,8 +3141,8 @@ namespace CNTK CNTK_API FunctionPtr Alias(const Variable& operand, const std::wstring& name = L""); /// - /// Creates a Block function that encapsulates a composite to create an opaque Function object that - /// appears as any other primitive function + /// Creates a Block Function that encapsulates a composite to create an opaque Function object that + /// appears as any other primitive Function /// CNTK_API FunctionPtr AsBlock(FunctionPtr&& composite, const std::vector>& argumentsMap, const std::wstring& blockOpName, const std::wstring& blockName = L""); @@ -3352,7 +3359,7 @@ namespace CNTK }; /// - /// Abstraction for learning a subset of parameters of a learnable function using first order gradient values + /// Abstraction for learning a subset of parameters of a learnable Function using first order gradient values /// For e.g momentum, AdaGrad, RMSProp etc. are different types of learners with their own algorithms for /// learning parameter values using first order gradients. /// @@ -3648,7 +3655,7 @@ namespace CNTK FunctionPtr Model() const { return m_model; } /// - /// Loss function that is used as the optimization criterion for learning the model's parameters. + /// Loss Function that is used as the optimization criterion for learning the model's parameters. /// FunctionPtr LossFunction() const { return m_lossFunction; } diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h b/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h index be8602023ad2..9136466eb211 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibraryInternals.h @@ -236,6 +236,9 @@ namespace CNTK CNTK_API void AlwaysAllowSettingDefaultDevice(); bool IsSettingDefaultDeviceAlwaysAllowed(); + CNTK_API void AllowRenamingFunctions(); + bool IsRenamingFunctionsAllowed(); + CNTK_API void SetAutomaticUnpackingOfPackedValues(bool disable); CNTK_API bool IsAutomaticUnpackingOfPackedValuesDisabled(); diff --git a/Source/CNTKv2LibraryDll/Common.cpp b/Source/CNTKv2LibraryDll/Common.cpp index e988cb1a7a81..523e9dad729e 100644 --- a/Source/CNTKv2LibraryDll/Common.cpp +++ b/Source/CNTKv2LibraryDll/Common.cpp @@ -49,6 +49,17 @@ namespace CNTK return s_alwaysAllowSettingDefaultDevice.load(); } + std::atomic s_allowRenamingFunctions(false); + void AllowRenamingFunctions() + { + s_allowRenamingFunctions.store(true); + } + + bool IsRenamingFunctionsAllowed() + { + return s_allowRenamingFunctions.load(); + } + std::atomic s_disableAutomaticUnpackingOfPackedValues(false); void SetAutomaticUnpackingOfPackedValues(bool disable) { diff --git a/Source/CNTKv2LibraryDll/Function.cpp b/Source/CNTKv2LibraryDll/Function.cpp index 072abcf15b35..a421240ece7c 100644 --- a/Source/CNTKv2LibraryDll/Function.cpp +++ b/Source/CNTKv2LibraryDll/Function.cpp @@ -68,6 +68,14 @@ namespace CNTK /*virtual*/ Function::~Function() {} + void Function::SetName(const std::wstring& name) + { + if (!Name().empty() && !Internal::IsRenamingFunctionsAllowed()) + InvalidArgument("Function::SetName: Illegal to set name of a Function with an existing name (%S)", Name().c_str()); + + m_name = name; + } + bool Function::IsBlock() const { auto blockFunction = dynamic_cast(this); diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index 26c7a2ddbf2b..5b3ee7a8c054 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -408,6 +408,8 @@ %ignore CNTK::Internal::IsReversingTensorShapesInErrorMessagesEnabled(); %ignore CNTK::Internal::AlwaysAllowSettingDefaultDevice(); %ignore CNTK::Internal::IsSettingDefaultDeviceAlwaysAllowed(); +%ignore CNTK::Internal::AllowRenamingFunctions(); +%ignore CNTK::Internal::IsRenamingFunctionsAllowed(); %ignore CNTK::Internal::SetAutomaticUnpackingOfPackedValues(bool disable); %ignore CNTK::Internal::IsAutomaticUnpackingOfPackedValuesDisabled(); %ignore CNTK::Internal::SetComputationNetworkTraceLevel(int traceLevel); diff --git a/bindings/python/cntk/cntk_py.i b/bindings/python/cntk/cntk_py.i index 8b0d882736a6..6283739b0cbb 100644 --- a/bindings/python/cntk/cntk_py.i +++ b/bindings/python/cntk/cntk_py.i @@ -64,6 +64,7 @@ // These aren't exported from the CNTK C++ library %ignore CNTK::Internal::IsReversingTensorShapesInErrorMessagesEnabled; %ignore CNTK::Internal::IsSettingDefaultDeviceAlwaysAllowed; +%ignore CNTK::Internal::IsRenamingFunctionsAllowed; %ignore CNTK::Internal::IsAutomaticUnpackingOfPackedValuesDisabled; %ignore CNTK::Internal::GetComputationNetworkTraceLevel; diff --git a/bindings/python/cntk/ops/functions.py b/bindings/python/cntk/ops/functions.py index c651682cc0cf..01d5b74abaa9 100644 --- a/bindings/python/cntk/ops/functions.py +++ b/bindings/python/cntk/ops/functions.py @@ -386,6 +386,18 @@ def name(self): ''' return super(Function, self).name() + @name.setter + def name(self, function_name): + ''' + Sets the name of this Function. + Setting the name of a Function is only allowed if the Function does not already have a name. + Calling this method, when this Function already has a name, results in an exception. + + Args: + function_name (`str`): name for this Function. + ''' + super(Function, self).set_name(function_name) + @property def op_name(self): ''' diff --git a/bindings/python/cntk/ops/tests/function_tests.py b/bindings/python/cntk/ops/tests/function_tests.py index d31eb9f1e3ad..fe1f99954053 100644 --- a/bindings/python/cntk/ops/tests/function_tests.py +++ b/bindings/python/cntk/ops/tests/function_tests.py @@ -126,3 +126,21 @@ def test_getting_output_from_non_existent_node(): with pytest.raises(ValueError): sum_output = times_node.forward({x: x0, y: y0}, sum_node.outputs) + +def test_set_name(): + x = input_variable((1,)) + y = input_variable((1,)) + x_plus_y = x + y + assert (x_plus_y.name == '') + x_plus_y.name = 'x_plus_y' + assert (x_plus_y.name == 'x_plus_y') + + x_plus_y_2 = plus(x, y, name='x_plus_y_2') + assert (x_plus_y_2.name == 'x_plus_y_2') + with pytest.raises(ValueError): + x_plus_y_2.name = 'x_plus_y_2_new' + + from ... import cntk_py + cntk_py.allow_renaming_functions() + + x_plus_y_2.name = 'x_plus_y_2_new' From 23a5cacea2cdce1bdf99fe6dfa64ece01d071cde Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Mon, 19 Dec 2016 15:42:35 +0100 Subject: [PATCH 052/120] complete Create{Batch|Sequence|BatchOfSequences} and other Creates at C# restore program.cs add missing changes --- bindings/common/CNTKValueExtend.i | 24 ++- .../CSEvalV2Library/CSEvalV2Library.csproj | 2 +- .../Properties/AssemblyInfo.cs | 2 +- bindings/csharp/Swig/CSharpBindings.vcxproj | 5 +- .../Swig/CSharpBindings.vcxproj.filters | 3 + bindings/csharp/Swig/cntk_cs.i | 199 +++++++++++------- 6 files changed, 157 insertions(+), 78 deletions(-) diff --git a/bindings/common/CNTKValueExtend.i b/bindings/common/CNTKValueExtend.i index 17f8b1974f1f..71679e12b326 100644 --- a/bindings/common/CNTKValueExtend.i +++ b/bindings/common/CNTKValueExtend.i @@ -4,16 +4,26 @@ // Value // %extend CNTK::Value { - static CNTK::ValuePtr CNTK::Value::CreateDenseFloat(const CNTK::NDShape& sampleShape, const std::vector>& sequences, + static CNTK::ValuePtr CNTK::Value::CreateDenseFloat(const CNTK::NDShape& sampleShape, const std::vector>& sequences, const CNTK::DeviceDescriptor& device, bool readOnly = false) { return CNTK::Value::Create(sampleShape, sequences, device, readOnly); } - static CNTK::ValuePtr CNTK::Value::CreateDenseDouble(const CNTK::NDShape& sampleShape, const std::vector>& sequences, + static CNTK::ValuePtr CNTK::Value::CreateDenseDouble(const CNTK::NDShape& sampleShape, const std::vector>& sequences, const CNTK::DeviceDescriptor& device, bool readOnly = false) { return CNTK::Value::Create(sampleShape, sequences, device, readOnly); } + static CNTK::ValuePtr CNTK::Value::CreateDenseFloat(const CNTK::NDShape& sampleShape, const std::vector>& sequences, + const std::vector& sequenceStartFlags, const CNTK::DeviceDescriptor& device, bool readOnly = false) { + return CNTK::Value::Create(sampleShape, sequences, sequenceStartFlags, device, readOnly); + } + + static CNTK::ValuePtr CNTK::Value::CreateDenseDouble(const CNTK::NDShape& sampleShape, const std::vector>& sequences, + const std::vector& sequenceStartFlags, const CNTK::DeviceDescriptor& device, bool readOnly = false) { + return CNTK::Value::Create(sampleShape, sequences, sequenceStartFlags, device, readOnly); + } + static CNTK::ValuePtr CNTK::Value::CreateOneHotFloat(size_t vocabularySize, const std::vector>& oneHotSequences, const CNTK::DeviceDescriptor& device, bool readOnly = false) { return CNTK::Value::Create(vocabularySize, oneHotSequences, device, readOnly); @@ -23,4 +33,14 @@ const CNTK::DeviceDescriptor& device, bool readOnly = false) { return CNTK::Value::Create(vocabularySize, oneHotSequences, device, readOnly); } + + static CNTK::ValuePtr CNTK::Value::CreateOneHotFloat(size_t vocabularySize, const std::vector>& oneHotSequences, + const std::vector& sequenceStartFlags, const CNTK::DeviceDescriptor& device, bool readOnly = false) { + return CNTK::Value::Create(vocabularySize, oneHotSequences, sequenceStartFlags, device, readOnly); + } + + static CNTK::ValuePtr CNTK::Value::CreateOneHotDouble(size_t vocabularySize, const std::vector>& oneHotSequences, + const std::vector& sequenceStartFlags, const CNTK::DeviceDescriptor& device, bool readOnly = false) { + return CNTK::Value::Create(vocabularySize, oneHotSequences, sequenceStartFlags, device, readOnly); + } } \ No newline at end of file diff --git a/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj b/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj index f7d4159854de..0a38a7866859 100644 --- a/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj +++ b/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj @@ -15,7 +15,7 @@ true Properties CNTK - CSEvalV2Library + CNTKLibraryManaged-2.0.dll v4.5 512 diff --git a/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs b/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs index f726a8135976..481e95f69457 100644 --- a/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs +++ b/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs @@ -5,7 +5,7 @@ // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. -[assembly: AssemblyTitle("CSEvalV2Library")] +[assembly: AssemblyTitle("CNTKLibraryManaged-2.0.dll")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] diff --git a/bindings/csharp/Swig/CSharpBindings.vcxproj b/bindings/csharp/Swig/CSharpBindings.vcxproj index 95052f240a03..15aa9fc7db42 100644 --- a/bindings/csharp/Swig/CSharpBindings.vcxproj +++ b/bindings/csharp/Swig/CSharpBindings.vcxproj @@ -116,11 +116,14 @@ - + + + + diff --git a/bindings/csharp/Swig/CSharpBindings.vcxproj.filters b/bindings/csharp/Swig/CSharpBindings.vcxproj.filters index 8aa4e1fb0f89..ac3c49e11456 100644 --- a/bindings/csharp/Swig/CSharpBindings.vcxproj.filters +++ b/bindings/csharp/Swig/CSharpBindings.vcxproj.filters @@ -33,4 +33,7 @@ Header Files + + + \ No newline at end of file diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index 26c7a2ddbf2b..538f41a1cc60 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -958,54 +958,115 @@ } // Create Value object from dense input: batch, sequence or batch of sequences. - - public static Value CreateBatch(NDShape shape, System.Collections.Generic.List batch, DeviceDescriptor computeDevice) + public static Value CreateBatch(NDShape shape, System.Collections.Generic.List batch, DeviceDescriptor device, bool readOnly = false) { - throw new System.ArgumentException("Not implemented yet."); + var input = new System.Collections.Generic.List>(); + var shapeSize = shape.TotalSize; + int i = 0; + System.Collections.Generic.List seq = null; + foreach (var element in batch) + { + if (++i % shapeSize == 0) + { + seq = new System.Collections.Generic.List(); + input.Add(seq); + } + seq.Add(element); + } + return Create(shape, input, device, readOnly); } - public static Value CreateSequence(NDShape shape, System.Collections.Generic.List sequence, - bool seqStartFalg, - DeviceDescriptor computeDevice) + public static Value CreateSequence(NDShape shape, + System.Collections.Generic.List sequence, + bool seqStartFlag, + DeviceDescriptor device, + bool readOnly = false) { - throw new System.ArgumentException("Not implemented yet."); + var input = new System.Collections.Generic.List>(1) {sequence}; + return Create(shape, input, new System.Collections.Generic.List(1) {seqStartFlag}, device, readOnly); } public static Value CreateBatchOfSequences(NDShape shape, System.Collections.Generic.List> batchOfSequences, - System.Collections.Generic.List seqStartFalg, - DeviceDescriptor computeDevice) + System.Collections.Generic.List seqStartFlags, + DeviceDescriptor device, + bool readOnly = false) + { + return Create(shape, batchOfSequences, seqStartFlags, device, readOnly); + } + + // Create Value object from OneHotVector input: batch, sequence or batch of sequences + public static Value CreateBatch(uint dimension, System.Collections.Generic.List batch, DeviceDescriptor device, bool readOnly = false) + { + // Is CreateBatch for OneHot really useful? + var input = new System.Collections.Generic.List>(); + batch.ForEach(element => input.Add(new System.Collections.Generic.List(1) {element})); + + return Create(dimension, input, device, readOnly); + } + + public static Value CreateSequence(uint dimension, + System.Collections.Generic.List sequence, + bool seqStartFlag, + DeviceDescriptor device, + bool readOnly = false) { - var dim = shape.TotalSize; + var input = new System.Collections.Generic.List>(1) {sequence}; + return Create(dimension, input, new System.Collections.Generic.List(1) {seqStartFlag}, device, readOnly); + } - // Todo: deal with seqStartFlag + public static Value CreateBatchOfSequences(uint dimension, + System.Collections.Generic.List> batchOfSequences, + System.Collections.Generic.List seqStartFlags, + DeviceDescriptor device, + bool readOnly = false) + { + return Create(dimension, batchOfSequences, seqStartFlags, device, readOnly); + } + + public static Value Create(NDShape sampleShape, + System.Collections.Generic.List sequences, + System.Collections.Generic.List sequenceStartFlags, + DeviceDescriptor device, + bool readOnly = false) + { + var seqVector = new NDArrayViewVector(sequences); + var startVector = new BoolVector(sequenceStartFlags); + return Create(sampleShape, seqVector, startVector, device, false); + } + + public static Value Create(NDShape sampleShape, + System.Collections.Generic.List sequences, + DeviceDescriptor device, + bool readOnly = false) + { + return Create(sampleShape, sequences, new System.Collections.Generic.List(0), device, readOnly); + } + + public static Value Create(NDShape sampleShape, + System.Collections.Generic.List> sequences, + System.Collections.Generic.List sequenceStartFlags, + DeviceDescriptor device, + bool readOnly = false) + { + var seqFlags = new BoolVector(sequenceStartFlags); if (typeof(T).Equals(typeof(float))) { var inputSeqVector = new FloatVectorVector(); - foreach (var seq in batchOfSequences) + foreach (var seq in sequences) { - if (seq.Count % dim != 0) - { - throw new System.ArgumentException("the number of data in sequences does not match the input dimension"); - } - var samples = new FloatVector(seq); - inputSeqVector.Add(samples); + inputSeqVector.Add(new FloatVector(seq)); } - return Value.CreateDenseFloat(shape, inputSeqVector, computeDevice); + return Value.CreateDenseFloat(sampleShape, inputSeqVector, seqFlags, device, readOnly); } else if (typeof(T).Equals(typeof(double))) { var inputSeqVector = new DoubleVectorVector(); - foreach (var seq in batchOfSequences) + foreach (var seq in sequences) { - if (seq.Count % dim != 0) - { - throw new System.ArgumentException("the number of data in sequences does not match the input dimension"); - } - var samples = new DoubleVector(seq); - inputSeqVector.Add(samples); + inputSeqVector.Add(new DoubleVector(seq)); } - return Value.CreateDenseDouble(shape, inputSeqVector, computeDevice); + return Value.CreateDenseDouble(sampleShape, inputSeqVector, seqFlags, device, readOnly); } else { @@ -1013,59 +1074,51 @@ } } - // Create Value object from OneHotVector input: batch, sequence or batch of sequences - public static Value CreateBatch(uint vacabSize, System.Collections.Generic.List batch, DeviceDescriptor computeDevice) - { - throw new System.ArgumentException("Not implemented yet."); - } - - public static Value CreateSequence(uint vacabSize, - System.Collections.Generic.List sequence, - bool seqStartFalg, - DeviceDescriptor computeDevice) + public static Value Create(NDShape sampleShape, + System.Collections.Generic.List> sequences, + DeviceDescriptor device, + bool readOnly = false) { - throw new System.ArgumentException("Not implemented yet."); + return Create(sampleShape, sequences, new System.Collections.Generic.List(0), device, readOnly); } - public static Value CreateBatchOfSequences(uint vacabSize, - System.Collections.Generic.List> batchOfSequences, - System.Collections.Generic.List seqStartFalg, - DeviceDescriptor computeDevice) + public static Value Create(uint dimension, + System.Collections.Generic.List> sequences, + System.Collections.Generic.List sequenceStartFlags, + DeviceDescriptor device, + bool readOnly = false) { - throw new System.NotImplementedException("Not implemented"); - } - - // Create Value object from sparse input: batch, sequence or batch of sequences. - public static Value CreateBatch(NDShape shape, - System.Collections.Generic.List data, - System.Collections.Generic.List indexes, - System.Collections.Generic.List nnzCounts, - DeviceDescriptor computeDevice) - { - throw new System.ArgumentException("Not implemented yet."); - } - - public static Value CreateSequence(NDShape shape, - System.Collections.Generic.List data, - System.Collections.Generic.List indexes, - System.Collections.Generic.List nnzCounts, - bool SequenceStartFlag, - DeviceDescriptor computeDevice) - { - throw new System.ArgumentException("Not implemented yet."); + var seqFlags = new BoolVector(sequenceStartFlags); + if (typeof(T).Equals(typeof(float))) + { + var inputSeqVector = new SizeTVectorVector(); + foreach (var seq in sequences) + { + inputSeqVector.Add(new SizeTVector(seq)); + } + return Value.CreateOneHotFloat(dimension, inputSeqVector, seqFlags, device, readOnly); + } + else if (typeof(T).Equals(typeof(double))) + { + var inputSeqVector = new SizeTVectorVector(); + foreach (var seq in sequences) + { + inputSeqVector.Add(new SizeTVector(seq)); + } + return Value.CreateOneHotDouble(dimension, inputSeqVector, seqFlags, device, readOnly); + } + else + { + throw new System.ArgumentException("The data type " + typeof(T).ToString() + " is not supported. Only float or double is supported by CNTK."); + } } - // Create Value based on sparse input - // Todo: could this be a extension to Value class?? - // Todo: use Variable instead of varName. VarName as extension method - public static Value CreateBatchOfSequences(NDShape shape, - System.Collections.Generic.List> data, - System.Collections.Generic.List> indexes, - System.Collections.Generic.List> nnzCounts, - System.Collections.Generic.List SequenceStartFlag, - DeviceDescriptor computeDevice) + public static Value Create(uint dimension, + System.Collections.Generic.List> sequences, + DeviceDescriptor device, + bool readOnly = false) { - throw new System.NotImplementedException("Not implemented"); + return Create(dimension, sequences, new System.Collections.Generic.List(0), device, readOnly); } %} From 5208999d209fcd2a6aa50466043c3595c772ad03 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 20 Dec 2016 13:20:46 +0100 Subject: [PATCH 053/120] complete CopyTo at C# --- bindings/csharp/CSEvalV2Example/Program.cs | 8 +-- .../csharp/CSEvalV2Library/ValueExtensions.cs | 66 ++++++++++++++++--- bindings/csharp/Swig/cntk_cs.i | 11 ++++ 3 files changed, 71 insertions(+), 14 deletions(-) diff --git a/bindings/csharp/CSEvalV2Example/Program.cs b/bindings/csharp/CSEvalV2Example/Program.cs index f0cb6c06d980..59661b00fa14 100644 --- a/bindings/csharp/CSEvalV2Example/Program.cs +++ b/bindings/csharp/CSEvalV2Example/Program.cs @@ -71,7 +71,7 @@ static void EvaluationWithDenseData(DeviceDescriptor device) // Get evaluate result as dense output var outputData = new List>(); Value outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(outputVar.Shape, outputData); + outputVal.CopyTo(outputVar, outputData); // Use case 2: Evaluate with batch of images Console.WriteLine("Evaluate batch of images"); @@ -100,7 +100,7 @@ static void EvaluationWithDenseData(DeviceDescriptor device) // Retrieve the evaluation result. outputData = new List>(); outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(outputVar.Shape, outputData); + outputVal.CopyTo(outputVar, outputData); // Output result Console.WriteLine("The number of sequences in the batch: " + outputData.Count); @@ -185,7 +185,7 @@ static void EvaluationWithOneHot(DeviceDescriptor device) // Get output result var outputData = new List>(); Value outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(vocabSize, outputData); + outputVal.CopyTo(outputVar, outputData); // Use case 2: evaluate batch of sequences using OneHot vector as input. @@ -230,7 +230,7 @@ static void EvaluationWithOneHot(DeviceDescriptor device) // Get evaluation result. outputData = new List>(); outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(vocabSize, outputData); + outputVal.CopyTo(outputVar, outputData); // output the result var numOfElementsInSample = vocabSize; diff --git a/bindings/csharp/CSEvalV2Library/ValueExtensions.cs b/bindings/csharp/CSEvalV2Library/ValueExtensions.cs index c8472721e85c..755d81f3948b 100644 --- a/bindings/csharp/CSEvalV2Library/ValueExtensions.cs +++ b/bindings/csharp/CSEvalV2Library/ValueExtensions.cs @@ -19,19 +19,52 @@ public static class ValueExtensions // The number of items contained in the outer list of 'sequences' is the number of sequences in the Value object. // Each element of the outer list represents a sequence. // Each sequence, represented by List, contains a variable number of samples. - // Each sample consits of a fixed number of elements with type of 'T'. The number of elements is determined by the sample shape. + // Each sample consits of a fixed number of elements with type of 'T'. The number of elements is determined by the variable shape. // The number of samples = the count of elements in List / the count of elements of the sample - // The sampleShape should match the shape of the Value object. + // The shape of the variable should match the shape of the Value object. // - public static void CopyTo(this Value value, NDShape sampelShape, List> sequences) + public static void CopyTo(this Value value, Variable sampleVariable, List> sequences) { - if ((value.GetDataType() == DataType.Float) && (!typeof(T).Equals(typeof(float))) || - (value.GetDataType() == DataType.Double) && (!typeof(T).Equals(typeof(double)))) + if (typeof(T).Equals(typeof(float))) { - throw new ArgumentException("The value type does not match the list type."); + if (value.GetDataType() != DataType.Float) + { + throw new ArgumentException("The value type does not match the list type."); + } + + var seqVec = new FloatVectorVector(); + value.CopyToFloat(sampleVariable, seqVec); + sequences.Clear(); + foreach (var seq in seqVec) + { + var seqList = seq as IEnumerable; + if (seqList == null) + throw new TypeAccessException("Cannot convert to the value type."); + sequences.Add(new List(seqList)); + } } + else if (typeof(T).Equals(typeof(double))) + { + if (value.GetDataType() != DataType.Double) + { + throw new ArgumentException("The value type does not match the list type."); + } - throw new Exception("Not implemented yet."); + var seqVec = new DoubleVectorVector(); + value.CopyToDouble(sampleVariable, seqVec); + sequences.Clear(); + foreach (var seq in seqVec) + { + var seqList = seq as IEnumerable; + if (seqList == null) + throw new TypeAccessException("Cannot convert to the value type."); + sequences.Add(new List(seqList)); + } + } + else + { + throw new ArgumentException("The value type does not match the list type."); + } } // @@ -40,12 +73,25 @@ public static void CopyTo(this Value value, NDShape sampelShape, List // The number of items contained in the outer list of 'sequences' is the number of sequences in the Value object. // Each element of the outer list represents a sequence. // Each sequence, represented by List, contains a variable number of samples. - // Each sample is represented by an index of the OneHot vector. The size of the OneHot vector is vocabularySize. + // Each sample is represented by an index of the OneHot vector. The size of the OneHot vector should match that defined in the variable. // The number of samples = the count of elements in List. // - public static void CopyTo(this Value value, uint vocabularySize, List> sequences) + public static void CopyTo(this Value value, Variable sampleVariable, List> sequences) { - throw new NotImplementedException("Not implemented"); + if (sampleVariable.Shape[0] != sampleVariable.Shape.TotalSize) + { + throw new ArgumentException("The sample variable's leading axis dimensionality must equal to the total size of the shape for sparse data"); + } + + var seqVec = new SizeTVectorVector(); + value.CopyTo(sampleVariable, seqVec); + + sequences.Clear(); + foreach(var seq in seqVec) + { + sequences.Add(new List(seq)); + } + return; } } } diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index 538f41a1cc60..99d556dba192 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -1122,6 +1122,17 @@ } %} +%extend CNTK::Value { + void CNTK::Value::CopyToFloat(const CNTK::Variable& sampleVariable, std::vector>& sequences) + { + return self->CopyTo(sampleVariable, sequences); + } + + void CNTK::Value::CopyToDouble(const CNTK::Variable& sampleVariable, std::vector>& sequences) + { + return self->CopyTo(sampleVariable, sequences); + } +} %include "CNTKLibraryInternals.h" %include "CNTKLibrary.h" From acb56de0d71cc4ebbd086dbbdbd80e00e1ee89b3 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 20 Dec 2016 13:34:12 +0100 Subject: [PATCH 054/120] change project names --- CNTK.sln | 4 ++-- bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj | 4 ++-- .../{CSEvalV2Library.csproj => CNTKLibraryManagedDll.csproj} | 2 +- bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs | 2 +- bindings/csharp/Swig/CSharpBindings.vcxproj | 1 + 5 files changed, 7 insertions(+), 6 deletions(-) rename bindings/csharp/CSEvalV2Library/{CSEvalV2Library.csproj => CNTKLibraryManagedDll.csproj} (99%) diff --git a/CNTK.sln b/CNTK.sln index 6e9fb31160e2..e8d40586bcf5 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1408,7 +1408,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "RNN", "RNN", "{6730F9BE-92A EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CSharp", "CSharp", "{1526F027-B007-472D-82E2-5A91340F3B62}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CSharpBindings", "bindings\csharp\Swig\CSharpBindings.vcxproj", "{277EBD9D-2504-49FA-AC72-59D5515130C3}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CNTKLibraryCSBinding", "bindings\csharp\Swig\CSharpBindings.vcxproj", "{277EBD9D-2504-49FA-AC72-59D5515130C3}" ProjectSection(ProjectDependencies) = postProject {E5606ECE-48CA-4464-BB12-09D81D02B9EF} = {E5606ECE-48CA-4464-BB12-09D81D02B9EF} EndProjectSection @@ -1418,7 +1418,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSEvalV2Example", "bindings {50EF9EE6-5018-453E-A063-F77044EF1A97} = {50EF9EE6-5018-453E-A063-F77044EF1A97} EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSEvalV2Library", "bindings\csharp\CSEvalV2Library\CSEvalV2Library.csproj", "{50EF9EE6-5018-453E-A063-F77044EF1A97}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryManagedDll", "bindings\csharp\CSEvalV2Library\CNTKLibraryManagedDll.csproj", "{50EF9EE6-5018-453E-A063-F77044EF1A97}" ProjectSection(ProjectDependencies) = postProject {277EBD9D-2504-49FA-AC72-59D5515130C3} = {277EBD9D-2504-49FA-AC72-59D5515130C3} EndProjectSection diff --git a/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj b/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj index e5991427b9d1..4e791ea780b1 100644 --- a/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj +++ b/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj @@ -71,9 +71,9 @@ - + {50ef9ee6-5018-453e-a063-f77044ef1a97} - CSEvalV2Library + CNTKLibraryManagedDll diff --git a/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj b/bindings/csharp/CSEvalV2Library/CNTKLibraryManagedDll.csproj similarity index 99% rename from bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj rename to bindings/csharp/CSEvalV2Library/CNTKLibraryManagedDll.csproj index 0a38a7866859..94f29d6bceb2 100644 --- a/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj +++ b/bindings/csharp/CSEvalV2Library/CNTKLibraryManagedDll.csproj @@ -15,7 +15,7 @@ true Properties CNTK - CNTKLibraryManaged-2.0.dll + CNTKLibraryManaged-2.0 v4.5 512 diff --git a/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs b/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs index 481e95f69457..701642e0d183 100644 --- a/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs +++ b/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs @@ -5,7 +5,7 @@ // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. -[assembly: AssemblyTitle("CNTKLibraryManaged-2.0.dll")] +[assembly: AssemblyTitle("CNTKLibraryManaged-2.0")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] diff --git a/bindings/csharp/Swig/CSharpBindings.vcxproj b/bindings/csharp/Swig/CSharpBindings.vcxproj index 15aa9fc7db42..ceff8c14ee9a 100644 --- a/bindings/csharp/Swig/CSharpBindings.vcxproj +++ b/bindings/csharp/Swig/CSharpBindings.vcxproj @@ -25,6 +25,7 @@ {277EBD9D-2504-49FA-AC72-59D5515130C3} CSharpBindings + CNTKLibraryCSBinding From ddd6bcbfcc81a3a9e9d6c2f239957f3a3a3d86e1 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 20 Dec 2016 14:36:13 +0100 Subject: [PATCH 055/120] rename CSEvalV2Libary to CNTKLibraryManagedDll --- .gitignore | 2 +- CNTK.sln | 2 +- .../CNTKLibraryManagedDll.csproj | 0 .../CntkBitmapExtensions.cs | 0 .../FunctionExtensions.cs | 0 .../Properties/AssemblyInfo.cs | 0 .../SwigProxyClasses/README.md | 0 .../ValueExtensions.cs | 0 bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj | 2 +- bindings/csharp/Swig/CSharpBindings.vcxproj | 2 +- 10 files changed, 4 insertions(+), 4 deletions(-) rename bindings/csharp/{CSEvalV2Library => CNTKLibraryManagedDll}/CNTKLibraryManagedDll.csproj (100%) rename bindings/csharp/{CSEvalV2Library => CNTKLibraryManagedDll}/CntkBitmapExtensions.cs (100%) rename bindings/csharp/{CSEvalV2Library => CNTKLibraryManagedDll}/FunctionExtensions.cs (100%) rename bindings/csharp/{CSEvalV2Library => CNTKLibraryManagedDll}/Properties/AssemblyInfo.cs (100%) rename bindings/csharp/{CSEvalV2Library => CNTKLibraryManagedDll}/SwigProxyClasses/README.md (100%) rename bindings/csharp/{CSEvalV2Library => CNTKLibraryManagedDll}/ValueExtensions.cs (100%) diff --git a/.gitignore b/.gitignore index defcd00b931e..fa97cf1c28ed 100644 --- a/.gitignore +++ b/.gitignore @@ -221,7 +221,7 @@ bindings/python/cntk/i_plus_i_0.mod # CSharp bindings bindings/csharp/Swig/cntk_cs_wrap.h bindings/csharp/Swig/cntk_cs_wrap.cxx -bindings/csharp/CSEvalV2Library/SwigProxyClasses/*.cs +bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/*.cs # Examples and Tutorials Examples/Image/DataSets/Pascal/VOC* diff --git a/CNTK.sln b/CNTK.sln index e8d40586bcf5..5c9272d3dbbc 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1418,7 +1418,7 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSEvalV2Example", "bindings {50EF9EE6-5018-453E-A063-F77044EF1A97} = {50EF9EE6-5018-453E-A063-F77044EF1A97} EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryManagedDll", "bindings\csharp\CSEvalV2Library\CNTKLibraryManagedDll.csproj", "{50EF9EE6-5018-453E-A063-F77044EF1A97}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryManagedDll", "bindings\csharp\CNTKLibraryManagedDll\CNTKLibraryManagedDll.csproj", "{50EF9EE6-5018-453E-A063-F77044EF1A97}" ProjectSection(ProjectDependencies) = postProject {277EBD9D-2504-49FA-AC72-59D5515130C3} = {277EBD9D-2504-49FA-AC72-59D5515130C3} EndProjectSection diff --git a/bindings/csharp/CSEvalV2Library/CNTKLibraryManagedDll.csproj b/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj similarity index 100% rename from bindings/csharp/CSEvalV2Library/CNTKLibraryManagedDll.csproj rename to bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj diff --git a/bindings/csharp/CSEvalV2Library/CntkBitmapExtensions.cs b/bindings/csharp/CNTKLibraryManagedDll/CntkBitmapExtensions.cs similarity index 100% rename from bindings/csharp/CSEvalV2Library/CntkBitmapExtensions.cs rename to bindings/csharp/CNTKLibraryManagedDll/CntkBitmapExtensions.cs diff --git a/bindings/csharp/CSEvalV2Library/FunctionExtensions.cs b/bindings/csharp/CNTKLibraryManagedDll/FunctionExtensions.cs similarity index 100% rename from bindings/csharp/CSEvalV2Library/FunctionExtensions.cs rename to bindings/csharp/CNTKLibraryManagedDll/FunctionExtensions.cs diff --git a/bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs b/bindings/csharp/CNTKLibraryManagedDll/Properties/AssemblyInfo.cs similarity index 100% rename from bindings/csharp/CSEvalV2Library/Properties/AssemblyInfo.cs rename to bindings/csharp/CNTKLibraryManagedDll/Properties/AssemblyInfo.cs diff --git a/bindings/csharp/CSEvalV2Library/SwigProxyClasses/README.md b/bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/README.md similarity index 100% rename from bindings/csharp/CSEvalV2Library/SwigProxyClasses/README.md rename to bindings/csharp/CNTKLibraryManagedDll/SwigProxyClasses/README.md diff --git a/bindings/csharp/CSEvalV2Library/ValueExtensions.cs b/bindings/csharp/CNTKLibraryManagedDll/ValueExtensions.cs similarity index 100% rename from bindings/csharp/CSEvalV2Library/ValueExtensions.cs rename to bindings/csharp/CNTKLibraryManagedDll/ValueExtensions.cs diff --git a/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj b/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj index 4e791ea780b1..bc570b3fde4e 100644 --- a/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj +++ b/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj @@ -71,7 +71,7 @@ - + {50ef9ee6-5018-453e-a063-f77044ef1a97} CNTKLibraryManagedDll diff --git a/bindings/csharp/Swig/CSharpBindings.vcxproj b/bindings/csharp/Swig/CSharpBindings.vcxproj index ceff8c14ee9a..2197ca948dc5 100644 --- a/bindings/csharp/Swig/CSharpBindings.vcxproj +++ b/bindings/csharp/Swig/CSharpBindings.vcxproj @@ -109,7 +109,7 @@ Document - $(SWIG_PATH)\swig.exe -c++ -csharp -DMSC_VER -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(SolutionDir)bindings\csharp\CSEvalV2Library\SwigProxyClasses -dllimport CSharpBindings cntk_cs.i + $(SWIG_PATH)\swig.exe -c++ -csharp -DMSC_VER -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(SolutionDir)bindings\csharp\CNTKLibraryManagedDll\SwigProxyClasses -dllimport CSharpBindings cntk_cs.i cntk_cs_wrap.cxx; cntk_cs_warp.h $(SolutionDir)Source\CNTKv2LibraryDll\API\CNTKLibrary.h;$(SolutionDir)bindings\common\CNTKValueExtend.i;std_unordered_map.i From f7a2172828871b2c459711fc5a997c6b2bf46b48 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 20 Dec 2016 14:57:56 +0100 Subject: [PATCH 056/120] move CSEvalV2Examples to the examples folder and move the project files to the Test folder --- CNTK.sln | 28 ++++++++----------- .../CNTKLibraryManagedClient}/Program.cs | 2 +- .../CNTKLibraryManagedClientTest}/App.config | 0 .../CNTKLibraryManagedClientTest.csproj | 12 +++++--- .../Properties/AssemblyInfo.cs | 4 +-- 5 files changed, 23 insertions(+), 23 deletions(-) rename {bindings/csharp/CSEvalV2Example => Examples/Evaluation/CNTKLibraryManagedClient}/Program.cs (99%) rename {bindings/csharp/CSEvalV2Example => Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest}/App.config (100%) rename bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj => Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/CNTKLibraryManagedClientTest.csproj (91%) rename {bindings/csharp/CSEvalV2Example => Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest}/Properties/AssemblyInfo.cs (91%) diff --git a/CNTK.sln b/CNTK.sln index 5c9272d3dbbc..f1a80f019eb4 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1413,11 +1413,6 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CNTKLibraryCSBinding", "bin {E5606ECE-48CA-4464-BB12-09D81D02B9EF} = {E5606ECE-48CA-4464-BB12-09D81D02B9EF} EndProjectSection EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSEvalV2Example", "bindings\csharp\CSEvalV2Example\CSEvalV2Example.csproj", "{3500A847-E024-4E7D-92DD-CC587C17460B}" - ProjectSection(ProjectDependencies) = postProject - {50EF9EE6-5018-453E-A063-F77044EF1A97} = {50EF9EE6-5018-453E-A063-F77044EF1A97} - EndProjectSection -EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryManagedDll", "bindings\csharp\CNTKLibraryManagedDll\CNTKLibraryManagedDll.csproj", "{50EF9EE6-5018-453E-A063-F77044EF1A97}" ProjectSection(ProjectDependencies) = postProject {277EBD9D-2504-49FA-AC72-59D5515130C3} = {277EBD9D-2504-49FA-AC72-59D5515130C3} @@ -1452,6 +1447,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerformanceProfilerDll", "S ProjectSection(ProjectDependencies) = postProject {86883653-8A61-4038-81A0-2379FAE4200A} = {86883653-8A61-4038-81A0-2379FAE4200A} EndProjectSection +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryManagedClientTest", "Tests\EndToEndTests\EvalClientTests\CNTKLibraryManagedClientTest\CNTKLibraryManagedClientTest.csproj", "{3500A847-E024-4E7D-92DD-CC587C17460B}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -1879,16 +1875,6 @@ Global {277EBD9D-2504-49FA-AC72-59D5515130C3}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64 {277EBD9D-2504-49FA-AC72-59D5515130C3}.Release|x64.ActiveCfg = Release|x64 {277EBD9D-2504-49FA-AC72-59D5515130C3}.Release|x64.Build.0 = Release|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Debug|x64.ActiveCfg = Debug|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Debug|x64.Build.0 = Debug|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Release_NoOpt|x64.ActiveCfg = Release_NoOpt|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Release|x64.ActiveCfg = Release|x64 - {3500A847-E024-4E7D-92DD-CC587C17460B}.Release|x64.Build.0 = Release|x64 {50EF9EE6-5018-453E-A063-F77044EF1A97}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 {50EF9EE6-5018-453E-A063-F77044EF1A97}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 {50EF9EE6-5018-453E-A063-F77044EF1A97}.Debug|x64.ActiveCfg = Debug|x64 @@ -1909,6 +1895,16 @@ Global {4B442D34-641A-4B37-9A4B-D18DBE28A979}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64 {4B442D34-641A-4B37-9A4B-D18DBE28A979}.Release|x64.ActiveCfg = Release|x64 {4B442D34-641A-4B37-9A4B-D18DBE28A979}.Release|x64.Build.0 = Release|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Debug_CpuOnly|x64.ActiveCfg = Debug_CpuOnly|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Debug_CpuOnly|x64.Build.0 = Debug_CpuOnly|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Debug|x64.ActiveCfg = Debug|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Debug|x64.Build.0 = Debug|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Release_CpuOnly|x64.ActiveCfg = Release_CpuOnly|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Release_CpuOnly|x64.Build.0 = Release_CpuOnly|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Release_NoOpt|x64.ActiveCfg = Release_NoOpt|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Release|x64.ActiveCfg = Release|x64 + {3500A847-E024-4E7D-92DD-CC587C17460B}.Release|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -2099,7 +2095,6 @@ Global {784A839C-762F-4A85-9EF1-A1E00546AD6C} = {D30B34AF-3618-4C55-900E-8F60A9F39E66} {6730F9BE-92AA-45F7-9F98-CD13E725CCA9} = {784A839C-762F-4A85-9EF1-A1E00546AD6C} {277EBD9D-2504-49FA-AC72-59D5515130C3} = {1526F027-B007-472D-82E2-5A91340F3B62} - {3500A847-E024-4E7D-92DD-CC587C17460B} = {1526F027-B007-472D-82E2-5A91340F3B62} {50EF9EE6-5018-453E-A063-F77044EF1A97} = {1526F027-B007-472D-82E2-5A91340F3B62} {2A95B23C-D91E-4DF9-B8F0-5E997608AB65} = {47755F2E-D674-4175-9E38-8EA053455072} {FB604F98-008F-45CD-B06E-42C30E121F13} = {2A95B23C-D91E-4DF9-B8F0-5E997608AB65} @@ -2107,5 +2102,6 @@ Global {39C3C8CA-9A8A-4733-ADBB-3E19D0F52528} = {2A95B23C-D91E-4DF9-B8F0-5E997608AB65} {CB4566F1-6C8F-4270-83EE-F6AED84EBB2B} = {39C3C8CA-9A8A-4733-ADBB-3E19D0F52528} {4B442D34-641A-4B37-9A4B-D18DBE28A979} = {DD043083-71A4-409A-AA91-F9C548DCF7EC} + {3500A847-E024-4E7D-92DD-CC587C17460B} = {05E45AF7-C069-4057-BC16-0A532D068CE4} EndGlobalSection EndGlobal diff --git a/bindings/csharp/CSEvalV2Example/Program.cs b/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs similarity index 99% rename from bindings/csharp/CSEvalV2Example/Program.cs rename to Examples/Evaluation/CNTKLibraryManagedClient/Program.cs index 59661b00fa14..caa3f810bf20 100644 --- a/bindings/csharp/CSEvalV2Example/Program.cs +++ b/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs @@ -13,7 +13,7 @@ using System.Threading.Tasks; using CNTK; -namespace CSEvalV2Example +namespace CNTKLibraryManagedClientTest { public class Program { diff --git a/bindings/csharp/CSEvalV2Example/App.config b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/App.config similarity index 100% rename from bindings/csharp/CSEvalV2Example/App.config rename to Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/App.config diff --git a/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/CNTKLibraryManagedClientTest.csproj similarity index 91% rename from bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj rename to Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/CNTKLibraryManagedClientTest.csproj index bc570b3fde4e..ebaedbe3e33f 100644 --- a/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj +++ b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/CNTKLibraryManagedClientTest.csproj @@ -14,8 +14,8 @@ false true Properties - CSEvalV2Example - CSEvalV2Example + CNTKLibraryManagedClientTest + CNTKLibraryManagedClientTest v4.5 512 @@ -64,18 +64,22 @@ - - + {50ef9ee6-5018-453e-a063-f77044ef1a97} CNTKLibraryManagedDll + + + Program.cs + + diff --git a/bindings/csharp/CSEvalV2Example/Properties/AssemblyInfo.cs b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/Properties/AssemblyInfo.cs similarity index 91% rename from bindings/csharp/CSEvalV2Example/Properties/AssemblyInfo.cs rename to Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/Properties/AssemblyInfo.cs index 98f75226085b..a76b93fbf13f 100644 --- a/bindings/csharp/CSEvalV2Example/Properties/AssemblyInfo.cs +++ b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/Properties/AssemblyInfo.cs @@ -5,11 +5,11 @@ // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. -[assembly: AssemblyTitle("CSEvalV2Example")] +[assembly: AssemblyTitle("CNTKLibraryManagedClientTest")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("CSEvalV2Example")] +[assembly: AssemblyProduct("CNTKLibraryManagedClientTest")] [assembly: AssemblyCopyright("Copyright © 2016")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] From d733aeea114a00bd516537a3baacb799560feb5e Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 20 Dec 2016 15:14:29 +0100 Subject: [PATCH 057/120] rename CSharpBinding to CNTKLibraryCSBidnging --- CNTK.sln | 2 +- .../CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj | 6 ------ .../csharp/CNTKLibraryManagedDll/Properties/AssemblyInfo.cs | 2 +- ...{CSharpBindings.vcxproj => CNTKLibraryCSBinding.vcxproj} | 6 +++--- ...vcxproj.filters => CNTKLibraryCSBinding.vcxproj.filters} | 0 5 files changed, 5 insertions(+), 11 deletions(-) rename bindings/csharp/Swig/{CSharpBindings.vcxproj => CNTKLibraryCSBinding.vcxproj} (96%) rename bindings/csharp/Swig/{CSharpBindings.vcxproj.filters => CNTKLibraryCSBinding.vcxproj.filters} (100%) diff --git a/CNTK.sln b/CNTK.sln index f1a80f019eb4..0fdef1604125 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1408,7 +1408,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "RNN", "RNN", "{6730F9BE-92A EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "CSharp", "CSharp", "{1526F027-B007-472D-82E2-5A91340F3B62}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CNTKLibraryCSBinding", "bindings\csharp\Swig\CSharpBindings.vcxproj", "{277EBD9D-2504-49FA-AC72-59D5515130C3}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CNTKLibraryCSBinding", "bindings\csharp\Swig\CNTKLibraryCSBinding.vcxproj", "{277EBD9D-2504-49FA-AC72-59D5515130C3}" ProjectSection(ProjectDependencies) = postProject {E5606ECE-48CA-4464-BB12-09D81D02B9EF} = {E5606ECE-48CA-4464-BB12-09D81D02B9EF} EndProjectSection diff --git a/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj b/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj index 94f29d6bceb2..9be205a2eb9d 100644 --- a/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj +++ b/bindings/csharp/CNTKLibraryManagedDll/CNTKLibraryManagedDll.csproj @@ -68,12 +68,6 @@ - - - {277ebd9d-2504-49fa-ac72-59d5515130c3} - CSharpBindings - - diff --git a/bindings/csharp/CNTKLibraryManagedDll/Properties/AssemblyInfo.cs b/bindings/csharp/CNTKLibraryManagedDll/Properties/AssemblyInfo.cs index 701642e0d183..e8bc26907420 100644 --- a/bindings/csharp/CNTKLibraryManagedDll/Properties/AssemblyInfo.cs +++ b/bindings/csharp/CNTKLibraryManagedDll/Properties/AssemblyInfo.cs @@ -9,7 +9,7 @@ [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("CSEvalV2Library")] +[assembly: AssemblyProduct("CNTKLibraryManagedDll")] [assembly: AssemblyCopyright("Copyright © 2016")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] diff --git a/bindings/csharp/Swig/CSharpBindings.vcxproj b/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj similarity index 96% rename from bindings/csharp/Swig/CSharpBindings.vcxproj rename to bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj index 2197ca948dc5..01fcb31cfcca 100644 --- a/bindings/csharp/Swig/CSharpBindings.vcxproj +++ b/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj @@ -24,7 +24,7 @@ {277EBD9D-2504-49FA-AC72-59D5515130C3} - CSharpBindings + CNTKLibraryCSBinding CNTKLibraryCSBinding @@ -52,7 +52,7 @@ false - + @@ -109,7 +109,7 @@ Document - $(SWIG_PATH)\swig.exe -c++ -csharp -DMSC_VER -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(SolutionDir)bindings\csharp\CNTKLibraryManagedDll\SwigProxyClasses -dllimport CSharpBindings cntk_cs.i + $(SWIG_PATH)\swig.exe -c++ -csharp -DMSC_VER -I$(SolutionDir)Source\CNTKv2LibraryDll\API -I$(SolutionDir)bindings\common -namespace CNTK -outdir $(SolutionDir)bindings\csharp\CNTKLibraryManagedDll\SwigProxyClasses -dllimport CNTKLibraryCSBinding cntk_cs.i cntk_cs_wrap.cxx; cntk_cs_warp.h $(SolutionDir)Source\CNTKv2LibraryDll\API\CNTKLibrary.h;$(SolutionDir)bindings\common\CNTKValueExtend.i;std_unordered_map.i diff --git a/bindings/csharp/Swig/CSharpBindings.vcxproj.filters b/bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj.filters similarity index 100% rename from bindings/csharp/Swig/CSharpBindings.vcxproj.filters rename to bindings/csharp/Swig/CNTKLibraryCSBinding.vcxproj.filters From 0b47f30ad016a84467b3a999090b60e280867d66 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Wed, 21 Dec 2016 13:05:04 +0100 Subject: [PATCH 058/120] remove check for PackedValue as it will be automatically unpacked when accessing Data/Mask; fix bugs --- Examples/Evaluation/CNTKLibraryManagedClient/Program.cs | 2 +- bindings/csharp/Swig/cntk_cs.i | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs b/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs index caa3f810bf20..0193ebe629c9 100644 --- a/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs +++ b/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs @@ -80,7 +80,7 @@ static void EvaluationWithDenseData(DeviceDescriptor device) var seqData = new List(); for (int sampleIndex = 0; sampleIndex < fileList.Count; sampleIndex++) { - bmp = new Bitmap(Bitmap.FromFile(fileList[sampleIndex++])); + bmp = new Bitmap(Bitmap.FromFile(fileList[sampleIndex])); resized = bmp.Resize((int)imageWidth, (int)imageHeight, true); resizedCHW = resized.ParallelExtractCHW(); // Aadd this sample to the data buffer. diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index 99d556dba192..16fc609c88d6 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -966,7 +966,7 @@ System.Collections.Generic.List seq = null; foreach (var element in batch) { - if (++i % shapeSize == 0) + if (i++ % shapeSize == 0) { seq = new System.Collections.Generic.List(); input.Add(seq); From 4b9cc9c0405556e318c6301b7950d012b0671e6a Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Wed, 21 Dec 2016 16:32:23 +0100 Subject: [PATCH 059/120] add CreateSequence without SeqStartFlag --- .../CNTKLibraryManagedClient/Program.cs | 3 ++- bindings/csharp/Swig/cntk_cs.i | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs b/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs index 0193ebe629c9..627e02fae649 100644 --- a/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs +++ b/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs @@ -88,7 +88,8 @@ static void EvaluationWithDenseData(DeviceDescriptor device) } // Create Value for the batch data. - inputVal = Value.CreateBatch(inputVar.Shape, seqData, device); + // inputVal = Value.CreateBatch(inputVar.Shape, seqData, device); + inputVal = Value.CreateSequence(inputVar.Shape, seqData, true, device); // Create input and output data map. inputDataMap[inputVar] = inputVal; diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index 16fc609c88d6..bbc75d791c5d 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -976,6 +976,14 @@ return Create(shape, input, device, readOnly); } + public static Value CreateSequence(NDShape shape, + System.Collections.Generic.List sequence, + DeviceDescriptor device, + bool readOnly = false) + { + return CreateSequence(shape, sequence, true, device, readOnly); + } + public static Value CreateSequence(NDShape shape, System.Collections.Generic.List sequence, bool seqStartFlag, @@ -1005,6 +1013,14 @@ return Create(dimension, input, device, readOnly); } + public static Value CreateSequence(uint dimension, + System.Collections.Generic.List sequence, + DeviceDescriptor device, + bool readOnly = false) + { + return CreateSequence(dimension, sequence, true, device, readOnly); + } + public static Value CreateSequence(uint dimension, System.Collections.Generic.List sequence, bool seqStartFlag, From a0554dfe9e046849dd65dc562ebbd7359dd3bc1a Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Wed, 21 Dec 2016 18:15:06 +0100 Subject: [PATCH 060/120] improve samples --- .../CNTKLibraryManagedClient/Program.cs | 81 +++++++++++-------- 1 file changed, 46 insertions(+), 35 deletions(-) diff --git a/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs b/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs index 627e02fae649..f243f79b5baa 100644 --- a/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs +++ b/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs @@ -28,6 +28,7 @@ public class Program static void EvaluationWithDenseData(DeviceDescriptor device) { const string outputName = "Plus2060_output"; + var inputDataMap = new Dictionary(); // Load the model. Function modelFunc = Function.LoadModel("z.model"); @@ -38,6 +39,9 @@ static void EvaluationWithDenseData(DeviceDescriptor device) // Get input variable. The model has only one single input. // The same way described above for output variable can be used here to get input variable by name. Variable inputVar = modelFunc.Arguments.Single(); + var outputDataMap = new Dictionary(); + Value inputVal, outputVal; + List> outputBuffer; // Get shape data for the input variable NDShape inputShape = inputVar.Shape; @@ -46,8 +50,6 @@ static void EvaluationWithDenseData(DeviceDescriptor device) uint imageChannels = inputShape[2]; uint imageSize = inputShape.TotalSize; - var outputDataMap = new Dictionary(); - // Use case 1: Evaluate with single image Console.WriteLine("Evaluate single image"); @@ -57,8 +59,7 @@ static void EvaluationWithDenseData(DeviceDescriptor device) List resizedCHW = resized.ParallelExtractCHW(); // Create input data map - var inputDataMap = new Dictionary(); - var inputVal = Value.CreateBatch(inputVar.Shape, resizedCHW, device); + inputVal = Value.CreateBatch(inputVar.Shape, resizedCHW, device); inputDataMap.Add(inputVar, inputVal); // Create ouput data map. Using null as Value to indicate using system allocated memory. @@ -69,12 +70,14 @@ static void EvaluationWithDenseData(DeviceDescriptor device) modelFunc.Evaluate(inputDataMap, outputDataMap, device); // Get evaluate result as dense output - var outputData = new List>(); - Value outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(outputVar, outputData); + outputBuffer = new List>(); + outputVal = outputDataMap[outputVar]; + outputVal.CopyTo(outputVar, outputBuffer); + + PrintOutput(outputVar.Shape.TotalSize, outputBuffer); // Use case 2: Evaluate with batch of images - Console.WriteLine("Evaluate batch of images"); + Console.WriteLine("\nEvaluate batch of images"); var fileList = new List() { "00000.png", "00001.png", "00002.png" }; var seqData = new List(); @@ -96,36 +99,14 @@ static void EvaluationWithDenseData(DeviceDescriptor device) outputDataMap[outputVar] = null; // Evaluate the model against the batch input - modelFunc.Evaluate(inputDataMap, outputDataMap, device); + //modelFunc.Evaluate(inputDataMap, outputDataMap, device); - // Retrieve the evaluation result. - outputData = new List>(); - outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(outputVar, outputData); + //// Retrieve the evaluation result. + //outputVal = outputDataMap[outputVar]; + //outputVal.CopyTo(outputVar, outputBuffer); // Output result - Console.WriteLine("The number of sequences in the batch: " + outputData.Count); - int seqNo = 0; - uint outputSampleSize = outputVar.Shape.TotalSize; - foreach(var seq in outputData) - { - Console.WriteLine(String.Format("Sequence {0} contains {1} samples.", seqNo++, seq.Count/outputSampleSize)); - uint i = 0; - uint sampleNo = 0; - foreach (var element in seq) - { - Console.Write(String.Format(" sample {0}: " + sampleNo)); - Console.Write(element); - if (++i % outputSampleSize == 0) - { - Console.WriteLine("."); - sampleNo++; - } - else - Console.WriteLine(","); - } - } - + PrintOutput(outputVar.Shape.TotalSize, outputBuffer); } // @@ -250,6 +231,36 @@ static void EvaluationWithOneHot(DeviceDescriptor device) } } + static void PrintOutput(uint sampleSize, List> outputBuffer) + { + Console.WriteLine("The number of sequences in the batch: " + outputBuffer.Count); + int seqNo = 0; + uint outputSampleSize = sampleSize; + foreach (var seq in outputBuffer) + { + Console.WriteLine(String.Format("Sequence {0} contains {1} samples.", seqNo++, seq.Count / outputSampleSize)); + uint i = 0; + uint sampleNo = 0; + foreach (var element in seq) + { + if (i++ % outputSampleSize == 0) + { + Console.Write(String.Format(" sample {0}: ", sampleNo)); + } + Console.Write(element); + if (i % outputSampleSize == 0) + { + Console.WriteLine("."); + sampleNo++; + } + else + { + Console.Write(","); + } + } + } + } + static void Main(string[] args) { Console.WriteLine("======== Evaluate model using C# ========"); From 1b122c484e766a9048494b449e0141332cc7e21c Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Tue, 3 Jan 2017 17:55:09 +0100 Subject: [PATCH 061/120] rename CNTKLibraryManagedClient to CNTKLibraryManagedExamples --- CNTK.sln | 2 +- .../Program.cs | 2 +- .../App.config | 0 .../CNTKLibraryManagedExamplesTest.csproj} | 6 +++--- .../Properties/AssemblyInfo.cs | 4 ++-- .../csharp/CNTKLibraryManagedDll/CntkBitmapExtensions.cs | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) rename Examples/Evaluation/{CNTKLibraryManagedClient => CNTKLibraryManagedExamples}/Program.cs (99%) rename Tests/EndToEndTests/EvalClientTests/{CNTKLibraryManagedClientTest => CNTKLibraryManagedExamplesTest}/App.config (100%) rename Tests/EndToEndTests/EvalClientTests/{CNTKLibraryManagedClientTest/CNTKLibraryManagedClientTest.csproj => CNTKLibraryManagedExamplesTest/CNTKLibraryManagedExamplesTest.csproj} (96%) rename Tests/EndToEndTests/EvalClientTests/{CNTKLibraryManagedClientTest => CNTKLibraryManagedExamplesTest}/Properties/AssemblyInfo.cs (91%) diff --git a/CNTK.sln b/CNTK.sln index 0fdef1604125..56deddd9aa34 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1447,7 +1447,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerformanceProfilerDll", "S ProjectSection(ProjectDependencies) = postProject {86883653-8A61-4038-81A0-2379FAE4200A} = {86883653-8A61-4038-81A0-2379FAE4200A} EndProjectSection -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryManagedClientTest", "Tests\EndToEndTests\EvalClientTests\CNTKLibraryManagedClientTest\CNTKLibraryManagedClientTest.csproj", "{3500A847-E024-4E7D-92DD-CC587C17460B}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryManagedExamplesTest", "Tests\EndToEndTests\EvalClientTests\CNTKLibraryManagedExamplesTest\CNTKLibraryManagedExamplesTest.csproj", "{3500A847-E024-4E7D-92DD-CC587C17460B}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs b/Examples/Evaluation/CNTKLibraryManagedExamples/Program.cs similarity index 99% rename from Examples/Evaluation/CNTKLibraryManagedClient/Program.cs rename to Examples/Evaluation/CNTKLibraryManagedExamples/Program.cs index f243f79b5baa..df46f9900dcb 100644 --- a/Examples/Evaluation/CNTKLibraryManagedClient/Program.cs +++ b/Examples/Evaluation/CNTKLibraryManagedExamples/Program.cs @@ -13,7 +13,7 @@ using System.Threading.Tasks; using CNTK; -namespace CNTKLibraryManagedClientTest +namespace CNTKLibraryManagedExampleTest { public class Program { diff --git a/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/App.config b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedExamplesTest/App.config similarity index 100% rename from Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/App.config rename to Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedExamplesTest/App.config diff --git a/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/CNTKLibraryManagedClientTest.csproj b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedExamplesTest/CNTKLibraryManagedExamplesTest.csproj similarity index 96% rename from Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/CNTKLibraryManagedClientTest.csproj rename to Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedExamplesTest/CNTKLibraryManagedExamplesTest.csproj index ebaedbe3e33f..7be632122306 100644 --- a/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/CNTKLibraryManagedClientTest.csproj +++ b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedExamplesTest/CNTKLibraryManagedExamplesTest.csproj @@ -14,8 +14,8 @@ false true Properties - CNTKLibraryManagedClientTest - CNTKLibraryManagedClientTest + CNTKLibraryManagedExamplesTest + CNTKLibraryManagedExamplesTest v4.5 512 @@ -76,7 +76,7 @@ - + Program.cs diff --git a/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/Properties/AssemblyInfo.cs b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedExamplesTest/Properties/AssemblyInfo.cs similarity index 91% rename from Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/Properties/AssemblyInfo.cs rename to Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedExamplesTest/Properties/AssemblyInfo.cs index a76b93fbf13f..29227a3519a3 100644 --- a/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedClientTest/Properties/AssemblyInfo.cs +++ b/Tests/EndToEndTests/EvalClientTests/CNTKLibraryManagedExamplesTest/Properties/AssemblyInfo.cs @@ -5,11 +5,11 @@ // General Information about an assembly is controlled through the following // set of attributes. Change these attribute values to modify the information // associated with an assembly. -[assembly: AssemblyTitle("CNTKLibraryManagedClientTest")] +[assembly: AssemblyTitle("CNTKLibraryManagedExamplesTest")] [assembly: AssemblyDescription("")] [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("CNTKLibraryManagedClientTest")] +[assembly: AssemblyProduct("CNTKLibraryManagedExamplesTest")] [assembly: AssemblyCopyright("Copyright © 2016")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] diff --git a/bindings/csharp/CNTKLibraryManagedDll/CntkBitmapExtensions.cs b/bindings/csharp/CNTKLibraryManagedDll/CntkBitmapExtensions.cs index 8918a39b9f94..24433de7ac82 100644 --- a/bindings/csharp/CNTKLibraryManagedDll/CntkBitmapExtensions.cs +++ b/bindings/csharp/CNTKLibraryManagedDll/CntkBitmapExtensions.cs @@ -14,7 +14,7 @@ namespace CNTK { - public static class CntkBitmapExtensions + public static class BitmapExtensions { /// /// Resizes an image From 5674d332218512172563dd80a036738232c76023 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Sat, 7 Jan 2017 10:27:23 +0100 Subject: [PATCH 062/120] update binding based on the latest API, update samples, add buidling vocabulary index in samples for the atis model. --- .../CNTKLibraryManagedExamples/Program.cs | 202 ++++++++++++++---- .../CNTKLibraryManagedDll/ValueExtensions.cs | 10 +- bindings/csharp/Swig/cntk_cs.i | 8 +- 3 files changed, 165 insertions(+), 55 deletions(-) diff --git a/Examples/Evaluation/CNTKLibraryManagedExamples/Program.cs b/Examples/Evaluation/CNTKLibraryManagedExamples/Program.cs index df46f9900dcb..a18eb1bb9485 100644 --- a/Examples/Evaluation/CNTKLibraryManagedExamples/Program.cs +++ b/Examples/Evaluation/CNTKLibraryManagedExamples/Program.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.Drawing; +using System.IO; using System.Linq; using System.Text; using System.Threading.Tasks; @@ -18,16 +19,16 @@ namespace CNTKLibraryManagedExampleTest public class Program { // - // The example shows + // The example shows // - how to load model. - // - how to prepare input data for a signle sample, a batch of samples in dense format. - // - how to prepare input and output data map - // - how to evaluate a model + // - how to prepare input data for a single sample. + // - how to prepare input and output data map. + // - how to evaluate a model. // - how to retrieve evaluation result and retrieve output data in dense format. // - static void EvaluationWithDenseData(DeviceDescriptor device) + static void EvaluationSingleImage(DeviceDescriptor device) { - const string outputName = "Plus2060_output"; + const string outputName = "Plus2060"; var inputDataMap = new Dictionary(); // Load the model. @@ -50,7 +51,6 @@ static void EvaluationWithDenseData(DeviceDescriptor device) uint imageChannels = inputShape[2]; uint imageSize = inputShape.TotalSize; - // Use case 1: Evaluate with single image Console.WriteLine("Evaluate single image"); // Image preprocessing to match input requirements of the model. @@ -72,13 +72,49 @@ static void EvaluationWithDenseData(DeviceDescriptor device) // Get evaluate result as dense output outputBuffer = new List>(); outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(outputVar, outputBuffer); + outputVal.CopyVariableValueTo(outputVar, outputBuffer); PrintOutput(outputVar.Shape.TotalSize, outputBuffer); + } + + // + // The example shows + // - how to load model. + // - how to prepare input data for a batch of samples. + // - how to prepare input and output data map. + // - how to evaluate a model. + // - how to retrieve evaluation result and retrieve output data in dense format. + // + static void EvaluationBatchOfImages(DeviceDescriptor device) + { + const string outputName = "Plus2060"; + var inputDataMap = new Dictionary(); + + // Load the model. + Function modelFunc = Function.LoadModel("z.model"); + + // Get output variable based on name + Variable outputVar = modelFunc.Outputs.Where(variable => string.Equals(variable.Name, outputName)).Single(); + + // Get input variable. The model has only one single input. + // The same way described above for output variable can be used here to get input variable by name. + Variable inputVar = modelFunc.Arguments.Single(); + var outputDataMap = new Dictionary(); + Value inputVal, outputVal; + List> outputBuffer; + + // Get shape data for the input variable + NDShape inputShape = inputVar.Shape; + uint imageWidth = inputShape[0]; + uint imageHeight = inputShape[1]; + uint imageChannels = inputShape[2]; + uint imageSize = inputShape.TotalSize; - // Use case 2: Evaluate with batch of images Console.WriteLine("\nEvaluate batch of images"); + Bitmap bmp, resized; + List resizedCHW; + var fileList = new List() { "00000.png", "00001.png", "00002.png" }; var seqData = new List(); for (int sampleIndex = 0; sampleIndex < fileList.Count; sampleIndex++) @@ -91,39 +127,45 @@ static void EvaluationWithDenseData(DeviceDescriptor device) } // Create Value for the batch data. - // inputVal = Value.CreateBatch(inputVar.Shape, seqData, device); - inputVal = Value.CreateSequence(inputVar.Shape, seqData, true, device); + inputVal = Value.CreateBatch(inputVar.Shape, seqData, device); - // Create input and output data map. - inputDataMap[inputVar] = inputVal; - outputDataMap[outputVar] = null; + // Create input data map. + inputDataMap.Add(inputVar, inputVal); + + // Create ouput data map. Using null as Value to indicate using system allocated memory. + // Alternatively, create a Value object and add it to the data map. + outputDataMap.Add(outputVar, null); // Evaluate the model against the batch input - //modelFunc.Evaluate(inputDataMap, outputDataMap, device); + modelFunc.Evaluate(inputDataMap, outputDataMap, device); + + // Retrieve the evaluation result. + outputBuffer = new List>(); + outputVal = outputDataMap[outputVar]; + outputVal.CopyVariableValueTo(outputVar, outputBuffer); - //// Retrieve the evaluation result. - //outputVal = outputDataMap[outputVar]; - //outputVal.CopyTo(outputVar, outputBuffer); - // Output result PrintOutput(outputVar.Shape.TotalSize, outputBuffer); } - // - // The example shows - // - how to use OneHot vector as input and output for evaluation - // The input data contains multiple sequences and each sequence contains multiple samples. - // There is only one non-zero value in each sample, so the sample can be represented by the index of this non-zero value - // - use variable name, instead of Variable, for as parameters for evaluate. // - static void EvaluationWithOneHot(DeviceDescriptor device) + // The example shows + // - how to load model. + // - how to prepare input data as sequence. + // - how to retrieve input and output variables by name. + // - how to prepare input and output data map. + // - how to evaluate a model. + // - how to retrieve evaluation result and retrieve output data in the one-hot vector format. + // + static void EvaluationSingleSequenceUsingOneHot(DeviceDescriptor device) { - // Todo: fill both index values - var vocabToIndex = new Dictionary(); - var indexToVocab = new Dictionary(); + var vocabToIndex = buildVocabIndex("ATIS.vocab"); + var indexToVocab = buildInvVocabIndex("ATIS.label"); Function myFunc = Function.LoadModel("atis.model"); + Console.WriteLine("Evaluate single sequence using one-hot vector"); + // Get input variable const string inputName = "features"; var inputVar = myFunc.Arguments.Where(variable => string.Equals(variable.Name, inputName)).Single(); @@ -154,7 +196,7 @@ static void EvaluationWithOneHot(DeviceDescriptor device) inputDataMap.Add(inputVar, inputValue); // Prepare output - const string outputName = "out.z_output"; + const string outputName = "out.z"; Variable outputVar = myFunc.Outputs.Where(variable => string.Equals(variable.Name, outputName)).Single(); // Create ouput data map. Using null as Value to indicate using system allocated memory. @@ -167,9 +209,48 @@ static void EvaluationWithOneHot(DeviceDescriptor device) // Get output result var outputData = new List>(); Value outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(outputVar, outputData); + outputVal.CopyVariableValueTo(outputVar, outputData); + + // output the result + var numOfElementsInSample = vocabSize; + uint seqNo = 0; + foreach (var seq in outputData) + { + Console.Write("Seq=" + seqNo + ":"); + foreach (var index in seq) + { + // get the word based on index + Console.Write(indexToVocab[index]); + } + Console.WriteLine(); + // next sequence. + seqNo++; + } + } + + // + // The example shows + // - how to load model. + // - how to prepare input data as batch of sequences with variable length. + // - how to retrieve input and output variables by name. + // - how to prepare input and output data map. + // - how to evaluate a model. + // - how to retrieve evaluation result and retrieve output data in the one-hot vector format. + // + static void EvaluationBatchOfSequencesUsingOneHot(DeviceDescriptor device) + { + var vocabToIndex = buildVocabIndex("ATIS.vocab"); + var indexToVocab = buildInvVocabIndex("ATIS.label"); + + Function myFunc = Function.LoadModel("atis.model"); + + Console.WriteLine("Evaluate batch of sequences with variable length using one-hot vector"); + + // Get input variable + const string inputName = "features"; + var inputVar = myFunc.Arguments.Where(variable => string.Equals(variable.Name, inputName)).Single(); - // Use case 2: evaluate batch of sequences using OneHot vector as input. + uint vocabSize = inputVar.Shape.TotalSize; // Prepare the input data. // Each sample is represented by an index to the onehot vector, so the index of the non-zero value of each sample is saved in the inner list. @@ -180,9 +261,11 @@ static void EvaluationWithOneHot(DeviceDescriptor device) var inputSentences = new List() { "BOS i would like to find a flight from charlotte to las vegas that makes a stop in st. louis EOS", - "BOS I want to book a flight from NewYork to Seattle EOS" + "BOS I want to book a flight from New York to Seattle EOS" }; + List seqData; + string[] substring; int numOfSequences = inputSentences.Count; for (int seqIndex = 0; seqIndex < numOfSequences; seqIndex++) { @@ -200,19 +283,27 @@ static void EvaluationWithOneHot(DeviceDescriptor device) } // Create the Value representing the batch data. - inputValue = Value.CreateBatchOfSequences(vocabSize, inputBatch, seqStartFlagBatch, DeviceDescriptor.CPUDevice); + var inputValue = Value.CreateBatchOfSequences(vocabSize, inputBatch, seqStartFlagBatch, DeviceDescriptor.CPUDevice); + + // Build input data map. + var inputDataMap = new Dictionary(); + inputDataMap.Add(inputVar, inputValue); - // Build input and output data map - inputDataMap[inputVar] = inputValue; - outputDataMap[outputVar] = null; + // Prepare output + const string outputName = "out.z"; + Variable outputVar = myFunc.Outputs.Where(variable => string.Equals(variable.Name, outputName)).Single(); + + // Create ouput data map. Using null as Value to indicate using system allocated memory. + var outputDataMap = new Dictionary(); + outputDataMap.Add(outputVar, null); // Evalaute the model myFunc.Evaluate(inputDataMap, outputDataMap, device); // Get evaluation result. - outputData = new List>(); - outputVal = outputDataMap[outputVar]; - outputVal.CopyTo(outputVar, outputData); + var outputData = new List>(); + var outputVal = outputDataMap[outputVar]; + outputVal.CopyVariableValueTo(outputVar, outputData); // output the result var numOfElementsInSample = vocabSize; @@ -231,7 +322,7 @@ static void EvaluationWithOneHot(DeviceDescriptor device) } } - static void PrintOutput(uint sampleSize, List> outputBuffer) + private static void PrintOutput(uint sampleSize, List> outputBuffer) { Console.WriteLine("The number of sequences in the batch: " + outputBuffer.Count); int seqNo = 0; @@ -261,16 +352,35 @@ static void PrintOutput(uint sampleSize, List> outputBuffer) } } + private static Dictionary buildVocabIndex(string filePath) + { + var vocab = new Dictionary(); + + string[] lines = File.ReadAllLines(filePath); + for (uint idx = 0; idx < (uint)lines.Count(); idx++) + vocab.Add(lines[idx], idx); + + return vocab; + } + + private static string[] buildInvVocabIndex(string filePath) + { + return File.ReadAllLines(filePath); + } + static void Main(string[] args) { Console.WriteLine("======== Evaluate model using C# ========"); - EvaluationWithDenseData(DeviceDescriptor.CPUDevice); - EvaluationWithOneHot(DeviceDescriptor.CPUDevice); + EvaluationSingleImage(DeviceDescriptor.CPUDevice); + EvaluationBatchOfImages(DeviceDescriptor.CPUDevice); + //TODO: Add examples with OneHot. + //EvaluationSingleSequenceUsingOneHot(DeviceDescriptor.CPUDevice); + //EvaluationBatchOfSequencesUsingOneHot(DeviceDescriptor.CPUDevice); - // For using GPU: - //EvaluationWithDenseData(DeviceDescriptor.GPUDevice(0)); - //EvaluationWithOneHot(DeviceDescriptor.GPUDevice(1)); + // TODO: using GPU. + //EvaluationSingleImage(DeviceDescriptor.GPUDevice(0)); + //EvaluationBatchOfImages(DeviceDescriptor.GPUDevice(0)); Console.WriteLine("======== Evaluation completes. ========"); } diff --git a/bindings/csharp/CNTKLibraryManagedDll/ValueExtensions.cs b/bindings/csharp/CNTKLibraryManagedDll/ValueExtensions.cs index 755d81f3948b..ca246b60ff5a 100644 --- a/bindings/csharp/CNTKLibraryManagedDll/ValueExtensions.cs +++ b/bindings/csharp/CNTKLibraryManagedDll/ValueExtensions.cs @@ -23,7 +23,7 @@ public static class ValueExtensions // The number of samples = the count of elements in List / the count of elements of the sample // The shape of the variable should match the shape of the Value object. // - public static void CopyTo(this Value value, Variable sampleVariable, List> sequences) + public static void CopyVariableValueTo(this Value value, Variable sampleVariable, List> sequences) { if (typeof(T).Equals(typeof(float))) { @@ -33,7 +33,7 @@ public static void CopyTo(this Value value, Variable sampleVariable, List(this Value value, Variable sampleVariable, List(this Value value, Variable sampleVariable, List. // - public static void CopyTo(this Value value, Variable sampleVariable, List> sequences) + public static void CopyVariableValueTo(this Value value, Variable sampleVariable, List> sequences) { if (sampleVariable.Shape[0] != sampleVariable.Shape.TotalSize) { @@ -84,7 +84,7 @@ public static void CopyTo(this Value value, Variable sampleVariable, List>& sequences) + void CNTK::Value::CopyVariableValueToFloat(const CNTK::Variable& sampleVariable, std::vector>& sequences) { - return self->CopyTo(sampleVariable, sequences); + return self->CopyVariableValueTo(sampleVariable, sequences); } - void CNTK::Value::CopyToDouble(const CNTK::Variable& sampleVariable, std::vector>& sequences) + void CNTK::Value::CopyVariableValueToDouble(const CNTK::Variable& sampleVariable, std::vector>& sequences) { - return self->CopyTo(sampleVariable, sequences); + return self->CopyVariableValueTo(sampleVariable, sequences); } } From bde15c4c3506e5c3ceebb3ad2ab174c6b55cec53 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Sat, 7 Jan 2017 10:59:42 +0100 Subject: [PATCH 063/120] fix solution file --- CNTK.sln | 1 + 1 file changed, 1 insertion(+) diff --git a/CNTK.sln b/CNTK.sln index 56deddd9aa34..f4c635ff6bf7 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1447,6 +1447,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "PerformanceProfilerDll", "S ProjectSection(ProjectDependencies) = postProject {86883653-8A61-4038-81A0-2379FAE4200A} = {86883653-8A61-4038-81A0-2379FAE4200A} EndProjectSection +EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CNTKLibraryManagedExamplesTest", "Tests\EndToEndTests\EvalClientTests\CNTKLibraryManagedExamplesTest\CNTKLibraryManagedExamplesTest.csproj", "{3500A847-E024-4E7D-92DD-CC587C17460B}" EndProject Global From 214eb09def1c4317138541423de96d36d464603e Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Sat, 7 Jan 2017 12:09:00 -0800 Subject: [PATCH 064/120] CNTK v2 library: Add data type inference for Parameters and Constants --- Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 10 ++++++++++ bindings/python/cntk/ops/__init__.py | 4 ++-- bindings/python/cntk/ops/tests/function_tests.py | 11 +++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index 88d7eed4b29b..b3abfeee3f53 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -113,6 +113,16 @@ namespace CNTK if (outputDataType == DataType::Unknown) outputDataType = firstKnownInputDataType; + // Propagate the data type to any input Parameters/Constants with unknown data type + if (inferDimensions && (outputDataType != DataType::Unknown)) + { + for (auto& input : inputs) + { + if ((input.GetDataType() == DataType::Unknown) && (input.IsConstant() || input.IsParameter())) + input.m_dataFields->m_dataType = outputDataType; + } + } + // We currently require that the inputs' dynamic axes, if any, match std::vector outputDynamicAxes; if ((op == PrimitiveOpType::SumAll) || diff --git a/bindings/python/cntk/ops/__init__.py b/bindings/python/cntk/ops/__init__.py index 2ff8416255b0..6cbf6dfa35c8 100644 --- a/bindings/python/cntk/ops/__init__.py +++ b/bindings/python/cntk/ops/__init__.py @@ -2130,7 +2130,7 @@ def placeholder_variable(shape=None, dynamic_axes=None, name=''): @typemap -def parameter(shape=None, init=None, device=None, name=''): +def parameter(shape=None, init=None, dtype=None, device=None, name=''): ''' It creates a parameter tensor. @@ -2168,7 +2168,7 @@ def parameter(shape=None, init=None, device=None, name=''): else: data_type = np.float32 else: - data_type = None + data_type = dtype return Parameter(shape, init, data_type, device, name) diff --git a/bindings/python/cntk/ops/tests/function_tests.py b/bindings/python/cntk/ops/tests/function_tests.py index fe1f99954053..e0f4b58a4edb 100644 --- a/bindings/python/cntk/ops/tests/function_tests.py +++ b/bindings/python/cntk/ops/tests/function_tests.py @@ -12,7 +12,9 @@ import pytest from ..functions import * from ...trainer import * +from ...initializer import glorot_uniform from .. import constant, parameter, input_variable, placeholder_variable, times, plus +from ... import InferredDimension from .ops_test_utils import compare_lists_of_np_arrays def test_variable_forwarding(): @@ -144,3 +146,12 @@ def test_set_name(): cntk_py.allow_renaming_functions() x_plus_y_2.name = 'x_plus_y_2_new' + + +def test_data_type_inference(): + x_float = input_variable((1,), dtype = np.float64) + param1 = parameter((InferredDimension, 1), init = glorot_uniform(), dtype = cntk_py.DataType_Unknown) + assert (param1.get_data_type() == cntk_py.DataType_Unknown) + + x_times_param1 = times(x_float, param1) + assert (param1.dtype == np.float64) From 0c0b5c145f45722905159d3eb6dd5d632e6afe0b Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Sun, 8 Jan 2017 13:17:49 -0800 Subject: [PATCH 065/120] Forgot a change I had on a different branch --- bindings/csharp/Swig/cntk_cs.i | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index cf4607f18073..1c8c4af4337c 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -257,6 +257,10 @@ const NDShape& lowerPad = {0}, const NDShape& upperPad = {0}, const std::wstring& name = L""); + + +%ignore CNTK::Unooling; + %ignore CNTK::BatchNormalization(const Variable& operand, const Variable& scale, const Variable& bias, From 077bf2387f1638e20705322bfc04dba967bf1386 Mon Sep 17 00:00:00 2001 From: Alexey Reznichenko Date: Mon, 9 Jan 2017 14:07:11 +0100 Subject: [PATCH 066/120] Fix broken dependency Add PerformanceProfilerDll as a build dependency to ReaderLib. --- CNTK.sln | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CNTK.sln b/CNTK.sln index f4c635ff6bf7..a5a432171422 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -134,8 +134,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ComputationNetworkLib", "So EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SGDLib", "Source\SGDLib\SGDLib.vcxproj", "{DE3C54E5-D7D0-47AF-A783-DFDCE59E7937}" ProjectSection(ProjectDependencies) = postProject - {16F14058-B116-49D9-8BA0-209F3AFFE849} = {16F14058-B116-49D9-8BA0-209F3AFFE849} {4B442D34-641A-4B37-9A4B-D18DBE28A979} = {4B442D34-641A-4B37-9A4B-D18DBE28A979} + {16F14058-B116-49D9-8BA0-209F3AFFE849} = {16F14058-B116-49D9-8BA0-209F3AFFE849} EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ParallelTraining", "ParallelTraining", "{5E666C53-2D82-49C9-9127-3FDDC321C741}" @@ -568,6 +568,9 @@ EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Speech", "Speech", "{FB7AF7B9-6BEA-459F-94D9-94D53916D2B6}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ReaderLib", "Source\Readers\ReaderLib\ReaderLib.vcxproj", "{F0A9637C-20DA-42F0-83D4-23B4704DE602}" + ProjectSection(ProjectDependencies) = postProject + {4B442D34-641A-4B37-9A4B-D18DBE28A979} = {4B442D34-641A-4B37-9A4B-D18DBE28A979} + EndProjectSection EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "AN4", "AN4", "{AC7BA8D3-B4C8-42A4-8507-B359BB6D49E8}" EndProject From 42a5aba1ca8d49444a139c70a40873178b63be65 Mon Sep 17 00:00:00 2001 From: Eldar Akchurin Date: Mon, 9 Jan 2017 12:31:45 +0100 Subject: [PATCH 067/120] Fixing number of buffers in the FrameMode packer of ImageReader --- Source/Readers/CompositeDataReader/CompositeDataReader.cpp | 2 +- Source/Readers/ImageReader/ImageReader.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Source/Readers/CompositeDataReader/CompositeDataReader.cpp b/Source/Readers/CompositeDataReader/CompositeDataReader.cpp index 0f90c19a4fd1..2e9a71030fa5 100644 --- a/Source/Readers/CompositeDataReader/CompositeDataReader.cpp +++ b/Source/Readers/CompositeDataReader/CompositeDataReader.cpp @@ -144,7 +144,7 @@ CompositeDataReader::CompositeDataReader(const ConfigParameters& config) : m_streams.push_back(stream); } - // Currently for prefetch we use two alternativing buffers, + // Currently for prefetch we use two alternating buffers, // same is the default. size_t numAlternatingBuffers = 2; diff --git a/Source/Readers/ImageReader/ImageReader.cpp b/Source/Readers/ImageReader/ImageReader.cpp index f658d5b2c692..462b474e786a 100644 --- a/Source/Readers/ImageReader/ImageReader.cpp +++ b/Source/Readers/ImageReader/ImageReader.cpp @@ -79,6 +79,7 @@ ImageReader::ImageReader(const ConfigParameters& config) m_packer = std::make_shared( m_sequenceEnumerator, m_streams, + 2 /* number of buffers*/, useLocalTimeline); } From f38f3087c157025340e4939372e9553cb9575897 Mon Sep 17 00:00:00 2001 From: Eldar Akchurin Date: Mon, 9 Jan 2017 13:28:50 +0100 Subject: [PATCH 068/120] Commenting out conv reference engine --- Source/Math/CuDnnConvolutionEngine.cu | 25 +++++++++++-------- bindings/python/cntk/ops/tests/kernel_test.py | 3 ++- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/Source/Math/CuDnnConvolutionEngine.cu b/Source/Math/CuDnnConvolutionEngine.cu index 4852183ee18c..36853caca86e 100644 --- a/Source/Math/CuDnnConvolutionEngine.cu +++ b/Source/Math/CuDnnConvolutionEngine.cu @@ -510,19 +510,22 @@ bool CuDnnConvolutionEngineFactory::IsSupported(DEVICEID_TYPE deviceId (poolKind == PoolKind::None || inputRank <= 3 && (kernelRank < 3 || kernel[2] == 1))); + return retVal; + + // TODO: This currently either causes a CUDA timeout or slows the whole machine down to a crawl (GPU). // cuDNN as of version 8.0 does not handle asymmetric padding for convolution correctly. We need to detect asymmetric // padding due to auto-padding and choose the reference convolution implementation instead - if (poolKind == PoolKind::None) // only for convolution, pooling seems fine - { - for (int i = 0; i < kernelRank; i++) - { - if (geometry->GetAutoPad(i)) - retVal = retVal && (kernel[i] % 2 != 0); // make sure kernel size is odd - else - retVal = retVal && (geometry->GetLowerPad(i) == geometry->GetUpperPad(i)); // lower pad is same as upper pad - } - } - return retVal; + //if (poolKind == PoolKind::None) // only for convolution, pooling seems fine + //{ + // for (int i = 0; i < kernelRank; i++) + // { + // if (geometry->GetAutoPad(i)) + // retVal = retVal && (kernel[i] % 2 != 0); // make sure kernel size is odd + // else + // retVal = retVal && (geometry->GetLowerPad(i) == geometry->GetUpperPad(i)); // lower pad is same as upper pad + // } + //} + //return retVal; } template class CuDnnConvolutionEngineFactory; diff --git a/bindings/python/cntk/ops/tests/kernel_test.py b/bindings/python/cntk/ops/tests/kernel_test.py index afe13057f02a..1e4372fcbc0e 100644 --- a/bindings/python/cntk/ops/tests/kernel_test.py +++ b/bindings/python/cntk/ops/tests/kernel_test.py @@ -78,7 +78,8 @@ def test_op_convolution_without_padding(convolution_map, convolution_input, devi [ 7, 8, 0]]]]) # result ] # this test handles convolution with asymmetric padding, in particular, with auto_padding is set to True -# and the kernel shape is even +# and the kernel shape is even +@pytest.mark.skip(reason="Reference model takes too long to run causing timeout, needs further investigation") @pytest.mark.parametrize("input_size, conv_size, result", ASYM_CONVOLUTION_DATA) def test_asym_convolution(input_size, conv_size, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] From a51e7fcab2a54fb039058bfad9d0905baf4d71cf Mon Sep 17 00:00:00 2001 From: Cha Zhang Date: Mon, 9 Jan 2017 15:24:22 -0800 Subject: [PATCH 069/120] Update cntk file. --- .../GoogLeNet/BrainScript/InceptionV3.cntk | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk b/Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk index 09763a0fe9a7..6fdbc58d4af7 100644 --- a/Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk +++ b/Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk @@ -89,17 +89,6 @@ Train = { gradientBits = 32 } } - - ParallelTrain = { - parallelizationMethod = BlockMomentumSGD - distributedMBReading = true - syncPerfStats = 5 - BlockMomentumSGD={ - syncPeriod = 120000 - resetSGDMomentum = true - useNesterovMomentum = true - } - } } reader = { From 6dc09565bb0b5e1a46354b9765b27923beab9985 Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 5 Jan 2017 15:47:42 +0100 Subject: [PATCH 070/120] CNTK_103B_MNIST_FeedForwardNetwork.ipynb: add test Also, let run MNIST tests against the default (end-user) data location. --- .../ConvNet/BrainScript/ConvNet_MNIST.cntk | 4 +- .../MLP/BrainScript/MLP_MNIST.cntk | 2 +- .../Classification/MLP/Python/SimpleMNIST.py | 20 ++--- Examples/Image/DataSets/MNIST/README.md | 2 +- .../Image/DataSets/MNIST/install_mnist.py | 2 + .../Image/GettingStarted/01_OneHidden.cntk | 2 +- Examples/Image/GettingStarted/02_OneConv.cntk | 2 +- .../GettingStarted/03_OneConvDropout.cntk | 2 +- .../Image/GettingStarted/04_OneConvBN.cntk | 2 +- .../Image/GettingStarted/05_OneConvRegr.cntk | 2 +- .../06_OneConvRegrMultiNode.cntk | 2 +- .../CMUDict/Python/Sequence2Sequence.py | 2 +- Scripts/README.md | 12 +-- .../NonSpatial/01_OneHidden.cntk | 2 +- .../NonSpatial/CNTK/run-test | 6 -- .../NonSpatial/CuDNN/run-test | 6 -- .../NonSpatial/run-test-common | 24 +----- ...CNTK_103B_MNIST_FeedForwardNetwork_test.py | 29 +++++++ .../CNTKv2Python/Examples/conftest.py | 8 +- .../01_OneHidden_ndl_deprecated.cntk | 1 - .../MNIST/01_OneHidden_ndl/run-test | 6 -- .../02_Convolution_ndl_deprecated.cntk | 1 - .../MNIST/02_Convolution_ndl/run-test | 7 -- .../03_ConvBatchNorm_ndl_deprecated.cntk | 1 - .../MNIST/03_ConvBatchNorm_ndl/run-test | 6 -- .../Image/Deprecated/MNIST/run-test-common | 26 +----- .../GettingStarted/01_OneHidden/run-test | 6 -- .../GettingStarted/01_OneHidden/testcases.yml | 2 +- .../Image/GettingStarted/02_OneConv/run-test | 6 -- .../GettingStarted/03_OneConvDropout/run-test | 6 -- .../03_OneConvDropout/testcases.yml | 2 +- .../GettingStarted/04_OneConvBN/run-test | 6 -- .../GettingStarted/04_OneConvBN/testcases.yml | 4 +- .../GettingStarted/05_OneConvRegr/run-test | 6 -- .../05_OneConvRegr/testcases.yml | 2 +- .../Image/GettingStarted/run-test-common | 27 ++----- Tools/get-datasets.sh | 39 +++++++++ Tutorials/CNTK_103A_MNIST_DataLoader.ipynb | 79 +++++-------------- .../CNTK_103B_MNIST_FeedForwardNetwork.ipynb | 49 ++++++------ 39 files changed, 161 insertions(+), 252 deletions(-) create mode 100644 Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py create mode 100755 Tools/get-datasets.sh diff --git a/Examples/Image/Classification/ConvNet/BrainScript/ConvNet_MNIST.cntk b/Examples/Image/Classification/ConvNet/BrainScript/ConvNet_MNIST.cntk index a2e0ca2b1b18..7909e4ce677a 100644 --- a/Examples/Image/Classification/ConvNet/BrainScript/ConvNet_MNIST.cntk +++ b/Examples/Image/Classification/ConvNet/BrainScript/ConvNet_MNIST.cntk @@ -55,7 +55,7 @@ trainNetwork = { minibatchSize = 64 maxEpochs = 40 learningRatesPerSample = 0.001*10:0.0005*10:0.0001 - dropoutRate = 0.5 + dropoutRate = 0.5 momentumAsTimeConstant = 0*5:1024 numMBsToShowResult = 500 @@ -63,7 +63,7 @@ trainNetwork = { reader = { readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" randomize = true keepDataInMemory = true diff --git a/Examples/Image/Classification/MLP/BrainScript/MLP_MNIST.cntk b/Examples/Image/Classification/MLP/BrainScript/MLP_MNIST.cntk index e5ddf72b619b..99c5d151827d 100644 --- a/Examples/Image/Classification/MLP/BrainScript/MLP_MNIST.cntk +++ b/Examples/Image/Classification/MLP/BrainScript/MLP_MNIST.cntk @@ -60,7 +60,7 @@ trainNetwork = { reader = { readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = { features = { dim = 784 ; format = "dense" } diff --git a/Examples/Image/Classification/MLP/Python/SimpleMNIST.py b/Examples/Image/Classification/MLP/Python/SimpleMNIST.py index f9cbc0bc583f..d209d3870fa5 100644 --- a/Examples/Image/Classification/MLP/Python/SimpleMNIST.py +++ b/Examples/Image/Classification/MLP/Python/SimpleMNIST.py @@ -14,7 +14,7 @@ from cntk.ops import input_variable, cross_entropy_with_softmax, classification_error, relu, element_times, constant abs_path = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(abs_path, "..", "..", "..", "..", "..", "Examples", "common")) +sys.path.append(os.path.join(abs_path, "..", "..", "..", "..", "common")) from nn import fully_connected_classifier_net, print_training_progress def check_path(path): @@ -51,12 +51,9 @@ def simple_mnist(debug_output=False): ce = cross_entropy_with_softmax(z, label) pe = classification_error(z, label) - try: - rel_path = os.path.join(os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'], - *"Image/MNIST/v0/Train-28x28_cntk_text.txt".split("/")) - except KeyError: - rel_path = os.path.join(abs_path, "..", "..", "..", "..", "..", "Examples", "Image", "DataSets", "MNIST", "Train-28x28_cntk_text.txt") - path = os.path.normpath(os.path.join(abs_path, rel_path)) + data_dir = os.path.join(abs_path, "..", "..", "..", "DataSets", "MNIST") + + path = os.path.normpath(os.path.join(data_dir, "Train-28x28_cntk_text.txt")) check_path(path) reader_train = create_reader(path, True, input_dim, num_output_classes) @@ -77,8 +74,6 @@ def simple_mnist(debug_output=False): num_minibatches_to_train = (num_samples_per_sweep * num_sweeps_to_train_with) / minibatch_size training_progress_output_freq = 500 - - if debug_output: training_progress_output_freq = training_progress_output_freq/4 @@ -88,12 +83,7 @@ def simple_mnist(debug_output=False): print_training_progress(trainer, i, training_progress_output_freq) # Load test data - try: - rel_path = os.path.join(os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'], - *"Image/MNIST/v0/Test-28x28_cntk_text.txt".split("/")) - except KeyError: - rel_path = os.path.join(abs_path, "..", "..", "..", "..", "..", "Examples", "Image", "DataSets", "MNIST", "Test-28x28_cntk_text.txt") - path = os.path.normpath(os.path.join(abs_path, rel_path)) + path = os.path.normpath(os.path.join(data_dir, "Test-28x28_cntk_text.txt")) check_path(path) reader_test = create_reader(path, False, input_dim, num_output_classes) diff --git a/Examples/Image/DataSets/MNIST/README.md b/Examples/Image/DataSets/MNIST/README.md index e8eb12bc93ca..1c7f42caa26e 100644 --- a/Examples/Image/DataSets/MNIST/README.md +++ b/Examples/Image/DataSets/MNIST/README.md @@ -7,7 +7,7 @@ downloaded and converted to CNTK-supported format by cd to this directory, Examp `python install_mnist.py` -After running the script, you will see two output files in the current folder: Train-28x28_cntk_text.txt and Test-28x28_cntk_text.txt. The total amount of disk space required is around `124`MB. You may now proceed to the [`GettingStarted`](../../GettingStarted) folder to play with this dataset. +After running the script, you will see two output files in the current folder: `Train-28x28_cntk_text.txt` and `Test-28x28_cntk_text.txt`. The total amount of disk space required is around `124`MB. You may now proceed to the [`GettingStarted`](../../GettingStarted) folder to play with this dataset. Further, we provide two advanced examples with MNIST. The first one is a [`Multi-Layer Perceptron network (MLP)`](../../Classification/MLP), which achieves about 1.5% error rate. The second one is a [`Convolutional Neural Network (ConvNet)`](../../Classification/ConvNet), which achieves about 0.5% error rate. These results are comparable to the best published results using these types of networks. diff --git a/Examples/Image/DataSets/MNIST/install_mnist.py b/Examples/Image/DataSets/MNIST/install_mnist.py index 235fa08cc1ac..321cb2f51d2a 100644 --- a/Examples/Image/DataSets/MNIST/install_mnist.py +++ b/Examples/Image/DataSets/MNIST/install_mnist.py @@ -1,7 +1,9 @@ from __future__ import print_function +import os import mnist_utils as ut if __name__ == "__main__": + os.chdir(os.path.abspath(os.path.dirname(__file__))) train = ut.load('http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz', 'http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz', 60000) print ('Writing train text file...') diff --git a/Examples/Image/GettingStarted/01_OneHidden.cntk b/Examples/Image/GettingStarted/01_OneHidden.cntk index 78805b343c41..792c75adb569 100644 --- a/Examples/Image/GettingStarted/01_OneHidden.cntk +++ b/Examples/Image/GettingStarted/01_OneHidden.cntk @@ -90,7 +90,7 @@ trainNetwork = { reader = { readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = { features = { dim = 784 ; format = "dense" } diff --git a/Examples/Image/GettingStarted/02_OneConv.cntk b/Examples/Image/GettingStarted/02_OneConv.cntk index aa989c503bd3..cb13a8e10b71 100644 --- a/Examples/Image/GettingStarted/02_OneConv.cntk +++ b/Examples/Image/GettingStarted/02_OneConv.cntk @@ -61,7 +61,7 @@ trainNetwork = { reader = { readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = { features = { dim = 784 ; format = "dense" } diff --git a/Examples/Image/GettingStarted/03_OneConvDropout.cntk b/Examples/Image/GettingStarted/03_OneConvDropout.cntk index 6ae99a223f30..6f2c00ed37d4 100644 --- a/Examples/Image/GettingStarted/03_OneConvDropout.cntk +++ b/Examples/Image/GettingStarted/03_OneConvDropout.cntk @@ -62,7 +62,7 @@ trainNetwork = { reader = { readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = { features = { dim = 784 ; format = "dense" } diff --git a/Examples/Image/GettingStarted/04_OneConvBN.cntk b/Examples/Image/GettingStarted/04_OneConvBN.cntk index 70e502fab05e..035c4220a85e 100644 --- a/Examples/Image/GettingStarted/04_OneConvBN.cntk +++ b/Examples/Image/GettingStarted/04_OneConvBN.cntk @@ -73,7 +73,7 @@ trainNetwork = { reader = { readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = { features = { dim = 784 ; format = "dense" } diff --git a/Examples/Image/GettingStarted/05_OneConvRegr.cntk b/Examples/Image/GettingStarted/05_OneConvRegr.cntk index 008bae4613e7..6674e918f1dd 100644 --- a/Examples/Image/GettingStarted/05_OneConvRegr.cntk +++ b/Examples/Image/GettingStarted/05_OneConvRegr.cntk @@ -62,7 +62,7 @@ trainNetwork = { reader = { readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = { features = { dim = 784 ; format = "dense" } diff --git a/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk b/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk index 795cd78fee97..89dccfc0606d 100644 --- a/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk +++ b/Examples/Image/GettingStarted/06_OneConvRegrMultiNode.cntk @@ -79,7 +79,7 @@ trainNetwork = { reader = { readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = { features = { dim = 784 ; format = "dense" } diff --git a/Examples/SequenceToSequence/CMUDict/Python/Sequence2Sequence.py b/Examples/SequenceToSequence/CMUDict/Python/Sequence2Sequence.py index e25b51c6a2f3..7f6119eeba40 100644 --- a/Examples/SequenceToSequence/CMUDict/Python/Sequence2Sequence.py +++ b/Examples/SequenceToSequence/CMUDict/Python/Sequence2Sequence.py @@ -16,7 +16,7 @@ from cntk.ops.functions import CloneMethod abs_path = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(abs_path, "..", "..", "..", "..", "Examples", "common")) +sys.path.append(os.path.join(abs_path, "..", "..", "..", "common")) from nn import LSTMP_component_with_self_stabilization, stabilize, linear_layer, print_training_progress # Given a vocab and tensor, print the output diff --git a/Scripts/README.md b/Scripts/README.md index 0c84af2e897f..10ba16ee1b24 100644 --- a/Scripts/README.md +++ b/Scripts/README.md @@ -30,11 +30,11 @@ For Example: ``` python Scripts/uci2ctf.py --input_file Examples/Image/MNIST/Data/Train-28x28.txt --features_start 1 --features_dim 784 --labels_start 0 --labels_dim 1 --num_labels 10 --output_file Examples/Image/MNIST/Data/Train-28x28_cntk_text.txt ``` -- `input_file` – original dataset in the (columnar) UCI format -- `features_start` – index of the first feature column (start parameter in the UCIFastReader config, see [here](https://github.com/Microsoft/CNTK/wiki/UCI-Fast-Reader) -- `features_dim` – number of feature columns (dim parameter in the UCIFastReader config) +- `input_file` - original dataset in the (columnar) UCI format +- `features_start` - index of the first feature column (start parameter in the UCIFastReader config, see [here](https://github.com/Microsoft/CNTK/wiki/UCI-Fast-Reader) +- `features_dim` - number of feature columns (dim parameter in the UCIFastReader config) - `labels_start` - index of the first label column -- `labels_dim` – number of label columns -- `num_labels` – number of possible label values (labelDim parameter in the UCIFastReader config) -- `output_file` – path and filename of the resulting dataset. +- `labels_dim` - number of label columns +- `num_labels` - number of possible label values (labelDim parameter in the UCIFastReader config) +- `output_file` - path and filename of the resulting dataset. diff --git a/Tests/EndToEndTests/BatchNormalization/NonSpatial/01_OneHidden.cntk b/Tests/EndToEndTests/BatchNormalization/NonSpatial/01_OneHidden.cntk index 2fd994497633..08824abfe237 100644 --- a/Tests/EndToEndTests/BatchNormalization/NonSpatial/01_OneHidden.cntk +++ b/Tests/EndToEndTests/BatchNormalization/NonSpatial/01_OneHidden.cntk @@ -39,7 +39,7 @@ train = [ reader = [ readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). + # See ../README.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = [ features = [ diff --git a/Tests/EndToEndTests/BatchNormalization/NonSpatial/CNTK/run-test b/Tests/EndToEndTests/BatchNormalization/NonSpatial/CNTK/run-test index 5e8e9c1ae362..504a75b37a3f 100755 --- a/Tests/EndToEndTests/BatchNormalization/NonSpatial/CNTK/run-test +++ b/Tests/EndToEndTests/BatchNormalization/NonSpatial/CNTK/run-test @@ -11,9 +11,3 @@ if [ $? != 0 ]; then fi cntkrun 01_OneHidden.cntk batchNormalizationEngine=cntk -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/BatchNormalization/NonSpatial/CuDNN/run-test b/Tests/EndToEndTests/BatchNormalization/NonSpatial/CuDNN/run-test index 885c49683b2f..df1ec81e6469 100755 --- a/Tests/EndToEndTests/BatchNormalization/NonSpatial/CuDNN/run-test +++ b/Tests/EndToEndTests/BatchNormalization/NonSpatial/CuDNN/run-test @@ -3,9 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 01_OneHidden.cntk batchNormalizationEngine=cudnn -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/BatchNormalization/NonSpatial/run-test-common b/Tests/EndToEndTests/BatchNormalization/NonSpatial/run-test-common index a7abea93d792..b28776179c74 100644 --- a/Tests/EndToEndTests/BatchNormalization/NonSpatial/run-test-common +++ b/Tests/EndToEndTests/BatchNormalization/NonSpatial/run-test-common @@ -4,23 +4,7 @@ ConfigDir=$TEST_DIR/.. -if [[ ! -d $TEST_DATA_DIR || ! -e $TEST_DATA_DIR/Train-28x28_cntk_text.txt || ! -e $TEST_DATA_DIR/Test-28x28_cntk_text.txt ]]; then - # Cannot find test data locally. - # Try external test data directory (not part of the CNTK repository) as an alternative. - if [[ -d "$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY" ]]; then - if [ "$OS" == "Windows_NT" ]; then - DataSourceDir=`cygpath -au $CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY`/Image/MNIST/v0 - else - DataSourceDir=$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY/Image/MNIST/v0 - fi - - # Copy the test data to the test run directory - DataDir=$TEST_RUN_DIR/TestData - mkdir $DataDir - cp -R $DataSourceDir/*_cntk_text.txt $DataDir || exit $? - Copied=1 - else - echo Error: cannot find data. Please see Examples/Image/MNIST/README.md for instructions to get it. - exit 1 - fi -fi +[[ -d $TEST_DATA_DIR && -f $TEST_DATA_DIR/Train-28x28_cntk_text.txt && -f $TEST_DATA_DIR/Test-28x28_cntk_text.txt ]] || { + echo Error: cannot find data. Please see Examples/Image/DataSets/MNIST/README.md for instructions to get it. + exit 1 +} diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py new file mode 100644 index 000000000000..518c974e58e4 --- /dev/null +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py @@ -0,0 +1,29 @@ +# Copyright (c) Microsoft. All rights reserved. + +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + +import os +import re +import numpy as np + +abs_path = os.path.dirname(os.path.abspath(__file__)) +notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_103B_MNIST_FeedForwardNetwork.ipynb") + +def test_cntk_103_mnist_feedforwardnetwork_noErrors(nb): + errors = [output for cell in nb.cells if 'outputs' in cell + for output in cell['outputs'] if output.output_type == "error"] + assert errors == [] + +expectedEvalErrorByDeviceId = { -1: 1.90, 0: 1.85 } + +def test_cntk_103_mnist_feedforwardnetwork_evalCorrect(nb, device_id): + testCell = [cell for cell in nb.cells + if cell.cell_type == 'code' and cell.source.find("print(\"Average test error:") != -1] + testCell = [cell for cell in nb.cells + if cell.cell_type == 'code' and re.search('trainer\.test_minibatch', cell.source)] + assert len(testCell) == 1 + m = re.match(r"Average test error: (?P\d+\.\d+)%\r?$", testCell[0].outputs[0]['text']) + # TODO tighten tolerances + assert np.isclose(float(m.group('actualEvalError')), expectedEvalErrorByDeviceId[device_id], atol=0.15) diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py b/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py index 360a453e6322..3885591dfaa2 100644 --- a/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py @@ -57,10 +57,16 @@ def pytest_generate_tests(metafunc): @pytest.fixture(scope='module') def nb(tmpdir_factory, request, device_id): - # TODO we need a way to inject device_id into the notebook import nbformat import os import subprocess + from cntk.ops.tests.ops_test_utils import cntk_device + from cntk.cntk_py import DeviceKind_GPU + # Pass along device_id type to child process + if cntk_device(device_id).type() == DeviceKind_GPU: + os.environ['TEST_DEVICE'] = 'gpu' + else: + os.environ['TEST_DEVICE'] = 'cpu' inPath = getattr(request.module, "notebook") outPath = str(tmpdir_factory.mktemp('notebook').join('out.ipynb')) assert os.path.isfile(inPath) diff --git a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/01_OneHidden_ndl/01_OneHidden_ndl_deprecated.cntk b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/01_OneHidden_ndl/01_OneHidden_ndl_deprecated.cntk index 16ac30880c61..e62098c60dec 100644 --- a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/01_OneHidden_ndl/01_OneHidden_ndl_deprecated.cntk +++ b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/01_OneHidden_ndl/01_OneHidden_ndl_deprecated.cntk @@ -88,7 +88,6 @@ train = [ reader = [ readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = [ features = [ diff --git a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/01_OneHidden_ndl/run-test b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/01_OneHidden_ndl/run-test index 17923e73d6c5..65ba3058d1df 100755 --- a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/01_OneHidden_ndl/run-test +++ b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/01_OneHidden_ndl/run-test @@ -3,9 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 01_OneHidden_ndl_deprecated.cntk "train=[SGD=[maxEpochs=3]] imageLayout=\"$imageLayout\" stderr=-" -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/02_Convolution_ndl/02_Convolution_ndl_deprecated.cntk b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/02_Convolution_ndl/02_Convolution_ndl_deprecated.cntk index 8895286eb014..ed2c8f98a490 100644 --- a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/02_Convolution_ndl/02_Convolution_ndl_deprecated.cntk +++ b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/02_Convolution_ndl/02_Convolution_ndl_deprecated.cntk @@ -52,7 +52,6 @@ train = [ reader = [ readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = [ features = [ diff --git a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/02_Convolution_ndl/run-test b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/02_Convolution_ndl/run-test index 081833a746d1..5e7f7e636efe 100755 --- a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/02_Convolution_ndl/run-test +++ b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/02_Convolution_ndl/run-test @@ -3,10 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 02_Convolution_ndl_deprecated.cntk "forceDeterministicAlgorithms=true train=[SGD=[maxEpochs=3]] imageLayout=\"$imageLayout\" stderr=-" - -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/03_ConvBatchNorm_ndl/03_ConvBatchNorm_ndl_deprecated.cntk b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/03_ConvBatchNorm_ndl/03_ConvBatchNorm_ndl_deprecated.cntk index b8ef48a317be..b84a233f6810 100644 --- a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/03_ConvBatchNorm_ndl/03_ConvBatchNorm_ndl_deprecated.cntk +++ b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/03_ConvBatchNorm_ndl/03_ConvBatchNorm_ndl_deprecated.cntk @@ -54,7 +54,6 @@ train = [ reader = [ readerType = "CNTKTextFormatReader" - # See ../REAMDE.md for details on getting the data (Train-28x28_cntk_text.txt). file = "$DataDir$/Train-28x28_cntk_text.txt" input = [ features = [ diff --git a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/03_ConvBatchNorm_ndl/run-test b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/03_ConvBatchNorm_ndl/run-test index e80f106e2e3c..e247b5664408 100755 --- a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/03_ConvBatchNorm_ndl/run-test +++ b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/03_ConvBatchNorm_ndl/run-test @@ -3,9 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 03_ConvBatchNorm_ndl_deprecated.cntk "train=[SGD=[maxEpochs=3]] imageLayout=\"$imageLayout\" stderr=-" -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/run-test-common b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/run-test-common index a89a4c54d5ea..032105503c89 100644 --- a/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/run-test-common +++ b/Tests/EndToEndTests/Examples/Image/Deprecated/MNIST/run-test-common @@ -4,25 +4,7 @@ ConfigDir=$TEST_DIR/ -if [[ ! -d $TEST_DATA_DIR || ! -e $TEST_DATA_DIR/Train-28x28_cntk_text.txt || ! -e $TEST_DATA_DIR/Test-28x28_cntk_text.txt ]]; then - # Cannot find test data locally. - # Try external test data directory (not part of the CNTK repository) as an alternative. - if [[ -d "$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY" ]]; then - if [ "$OS" == "Windows_NT" ]; then - DataSourceDir=`cygpath -au $CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY`/Image/MNIST/v0 - else - DataSourceDir=$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY/Image/MNIST/v0 - fi - - # Copy the test data to the test run directory - DataDir=$TEST_RUN_DIR/TestData - mkdir $DataDir - cp -R $DataSourceDir/*_cntk_text.txt $DataDir || exit $? - Copied=1 - else - echo Error: cannot find data. Please see Examples/Image/MNIST/README.md for instructions to get it. - exit 1 - fi -fi - - +[[ -d $TEST_DATA_DIR && -f $TEST_DATA_DIR/Train-28x28_cntk_text.txt && -f $TEST_DATA_DIR/Test-28x28_cntk_text.txt ]] || { + echo Error: cannot find data. Please see Examples/Image/DataSets/MNIST/README.md for instructions to get it. + exit 1 +} diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/01_OneHidden/run-test b/Tests/EndToEndTests/Examples/Image/GettingStarted/01_OneHidden/run-test index 8b92cfadf662..ce420ce59d70 100755 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/01_OneHidden/run-test +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/01_OneHidden/run-test @@ -3,9 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 01_OneHidden.cntk "forceDeterministicAlgorithms=true stderr=- trainNetwork=[SGD=[maxEpochs=3]]" -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/01_OneHidden/testcases.yml b/Tests/EndToEndTests/Examples/Image/GettingStarted/01_OneHidden/testcases.yml index 7464b70a4841..c5108708e5c7 100644 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/01_OneHidden/testcases.yml +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/01_OneHidden/testcases.yml @@ -1,4 +1,4 @@ -dataDir: ../../../../../../Examples/Image/DataSets +dataDir: ../../../../../../Examples/Image/DataSets/MNIST tags: - bvt-e (build_sku=='gpu') and (device=='gpu') and (flavor=='release') diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/02_OneConv/run-test b/Tests/EndToEndTests/Examples/Image/GettingStarted/02_OneConv/run-test index 25839897b447..695d2afd4088 100755 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/02_OneConv/run-test +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/02_OneConv/run-test @@ -3,9 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 02_OneConv.cntk "forceDeterministicAlgorithms=true stderr=- trainNetwork=[SGD=[maxEpochs=3]]" -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/03_OneConvDropout/run-test b/Tests/EndToEndTests/Examples/Image/GettingStarted/03_OneConvDropout/run-test index 7a3096b963ca..f6738cc1bd00 100755 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/03_OneConvDropout/run-test +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/03_OneConvDropout/run-test @@ -3,9 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 03_OneConvDropout.cntk "forceDeterministicAlgorithms=true stderr=- trainNetwork=[SGD=[maxEpochs=3]]" -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/03_OneConvDropout/testcases.yml b/Tests/EndToEndTests/Examples/Image/GettingStarted/03_OneConvDropout/testcases.yml index 7464b70a4841..c5108708e5c7 100644 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/03_OneConvDropout/testcases.yml +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/03_OneConvDropout/testcases.yml @@ -1,4 +1,4 @@ -dataDir: ../../../../../../Examples/Image/DataSets +dataDir: ../../../../../../Examples/Image/DataSets/MNIST tags: - bvt-e (build_sku=='gpu') and (device=='gpu') and (flavor=='release') diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/04_OneConvBN/run-test b/Tests/EndToEndTests/Examples/Image/GettingStarted/04_OneConvBN/run-test index c5c1fc1b1adb..bb690601f3c9 100755 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/04_OneConvBN/run-test +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/04_OneConvBN/run-test @@ -3,9 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 04_OneConvBN.cntk "forceDeterministicAlgorithms=true stderr=- trainNetwork=[SGD=[maxEpochs=3]]" -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/04_OneConvBN/testcases.yml b/Tests/EndToEndTests/Examples/Image/GettingStarted/04_OneConvBN/testcases.yml index d20f47245100..7e5e3f9b1f3f 100644 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/04_OneConvBN/testcases.yml +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/04_OneConvBN/testcases.yml @@ -1,4 +1,4 @@ -dataDir: ../../../../../../Examples/Image/DataSets +dataDir: ../../../../../../Examples/Image/DataSets/MNIST tags: # In BVT, run Release GPU @@ -36,4 +36,4 @@ testCases: patterns: - "Final Results: Minibatch[{{integer}}-{{integer}}]" # - errs = {{float,tolerance=1%}}% * {{integer}} -# - ce = {{float,tolerance=1%}} * {{integer}} \ No newline at end of file +# - ce = {{float,tolerance=1%}} * {{integer}} diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/05_OneConvRegr/run-test b/Tests/EndToEndTests/Examples/Image/GettingStarted/05_OneConvRegr/run-test index 80525628950a..e085ae226d66 100755 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/05_OneConvRegr/run-test +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/05_OneConvRegr/run-test @@ -3,9 +3,3 @@ . $TEST_DIR/../run-test-common cntkrun 05_OneConvRegr.cntk "forceDeterministicAlgorithms=true stderr=- trainNetwork=[SGD=[maxEpochs=3]]" -ExitCode=$? - -# Delete the test data if copied -[[ "$Copied" -eq "1" ]] && rm -rf "$DataDir" - -exit $ExitCode diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/05_OneConvRegr/testcases.yml b/Tests/EndToEndTests/Examples/Image/GettingStarted/05_OneConvRegr/testcases.yml index 7b4c3fbb7480..f0d83b17d64d 100644 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/05_OneConvRegr/testcases.yml +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/05_OneConvRegr/testcases.yml @@ -1,4 +1,4 @@ -dataDir: ../../../../../../Examples/Image/DataSets +dataDir: ../../../../../../Examples/Image/DataSets/MNIST tags: - bvt-e (build_sku=='gpu') and (device=='gpu') and (flavor=='release') diff --git a/Tests/EndToEndTests/Examples/Image/GettingStarted/run-test-common b/Tests/EndToEndTests/Examples/Image/GettingStarted/run-test-common index 9ac152649d24..01cb320f265d 100644 --- a/Tests/EndToEndTests/Examples/Image/GettingStarted/run-test-common +++ b/Tests/EndToEndTests/Examples/Image/GettingStarted/run-test-common @@ -1,6 +1,6 @@ #!/bin/bash -. $TEST_DIR/../../run-test-common # this will download all data sets necessary for testing +. $TEST_DIR/../../run-test-common export MKL_NUM_THREADS=4 export MKL_CBWR=COMPATIBLE @@ -8,24 +8,7 @@ export OMP_NUM_THREADS=1 ConfigDir=$TEST_DIR/../../../../../../Examples/Image/GettingStarted -if [[ ! -d $TEST_DATA_DIR || ! -e $TEST_DATA_DIR/Train-28x28_cntk_text.txt || ! -e $TEST_DATA_DIR/Test-28x28_cntk_text.txt ]]; then - # Cannot find test data locally. - # Try external test data directory (not part of the CNTK repository) as an alternative. - if [[ -d "$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY" ]]; then - if [ "$OS" == "Windows_NT" ]; then - DataSourceDir=`cygpath -au $CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY`/Image/MNIST/v0 - else - DataSourceDir=$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY/Image/MNIST/v0 - fi - - # Copy the test data to the test run directory - DataDir=$TEST_RUN_DIR/TestData - mkdir $DataDir - cp -R $DataSourceDir/*_cntk_text.txt $DataDir || exit $? - Copied=1 - else - echo Error: cannot find data. Please see Examples/Image/MNIST/README.md for instructions to get it. - exit 1 - fi -fi - +[[ -d $TEST_DATA_DIR && -f $TEST_DATA_DIR/Train-28x28_cntk_text.txt && -f $TEST_DATA_DIR/Test-28x28_cntk_text.txt ]] || { + echo Error: cannot find data. Please see Examples/Image/DataSets/MNIST/README.md for instructions to get it. + exit 1 +} diff --git a/Tools/get-datasets.sh b/Tools/get-datasets.sh new file mode 100755 index 000000000000..9cc4d4fe7fd0 --- /dev/null +++ b/Tools/get-datasets.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE file in the project root for full license information. +# +# Internal use: populate data sets to their default location + +set -x -e -o pipefail + +# Configuration data +declare -A dataSetMap +dataSetMap=( + ["MNIST-v0.tar.gz"]="Examples/Image/DataSets/MNIST" +) + +# Change directory to repository root +SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" +cd "$SCRIPT_DIR/.." + +# Validate source directory +[ -n "$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY" ] +if [ "$OS" == "Windows_NT" ]; then + DATADIR="$(cygpath -au "$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY")" +else + DATADIR="$CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY" +fi + +[ -d "$DATADIR" ] +DATADIR+=/DataSets +[ -d "$DATADIR" ] + +# Un-tar datasets (note: tar should automatically uncompress) +for dataSet in "${!dataSetMap[@]}"; do + archive="$DATADIR/$dataSet" + outDir="${dataSetMap[$dataSet]}" + [ -f "$archive" ] + [ -d "$outDir" ] + tar -xf "$archive" -C "$outDir" +done diff --git a/Tutorials/CNTK_103A_MNIST_DataLoader.ipynb b/Tutorials/CNTK_103A_MNIST_DataLoader.ipynb index 29df97de56a2..f66668be3247 100644 --- a/Tutorials/CNTK_103A_MNIST_DataLoader.ipynb +++ b/Tutorials/CNTK_103A_MNIST_DataLoader.ipynb @@ -20,7 +20,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "collapsed": false }, @@ -57,7 +57,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": { "collapsed": true }, @@ -130,28 +130,11 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Downloading train data\n", - "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n", - "Done.\n", - "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n", - "Done.\n", - "Downloading test data\n", - "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n", - "Done.\n", - "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n", - "Done.\n" - ] - } - ], + "outputs": [], "source": [ "# URLs for the train image and labels data\n", "url_train_image = 'http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz'\n", @@ -179,29 +162,11 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Image Label: 3\n" - ] - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWEAAAFfCAYAAACfj30KAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJztnel24zrPrOnZztC9v/u/yXd3J56H82MfqEslgKQcu+Uk\n9azFJUpybFmOShAIgKPL5ZKEEEIMw3joAxBCiO+MRFgIIQZEIiyEEAMiERZCiAGRCAshxIBIhIUQ\nYkAkwkIIMSASYSGEGBCJsBBCDIhEWAghBmQ69AH8f5Q7LYT4ioxKL5AlLIQQAyIRFkKIAZEICyHE\ngEiEhRBiQCTCQggxIBJhIYQYEImwEEIMiERYCCEGRCIshBADIhEWQogBkQgLIcSASISFEGJAJMJC\nCDEgEmEhhBgQibAQQgyIRFgIIQZEIiyEEAMiERZCiAGRCAshxIBIhIUQYkAkwkIIMSASYSGEGBCJ\nsBBCDIhEWAghBkQiLIQQAyIRFkKIAZEICyHEgEiEhRBiQCTCQggxIBJhIYQYEImwEEIMiERYCCEG\nRCIshBADIhEWQogBkQgLIcSASISFEGJAJMJCCDEgEmEhhBgQibAQQgyIRFgIIQZEIiyEEAMiERZC\niAGRCAshxIBIhIUQYkAkwkIIMSASYSGEGBCJsBBCDIhEWAghBmQ69AEI0YfL5dJ7v23DfTXbouXf\nZjQaZbf3XXrvmdsn7otEWHxackJq/fP5nC6XS6vltpWW1u5JrZiOx+M0Go3SeDxu9XlpfWz2Pl4T\nfxeJsPh0eBaq1z+fz73a6XTq9L3lPUU4J5K8bTKZpPF4nCaTSdjHpSfYLNri7yMRFp8StkrZsjXr\n9XQ6tQSUG24/Ho9Vy/P5fLfvhSLrWbC4fTKZpOl02rTcugmuCbL1L5dLGo/Hrc8WfxeJsPi0eMKL\nAnw+n1sCmutbOxwO7hL7p9Ppbt8p51ZAER6Px2k6nabZbNZq0TazkLFdLpc0mUxSSv8J8FA+7++O\nRFh8OiLR5T6KbCSuuNzv9+lwOLT63vLeIowNRZe3zefzsM1ms9Y6Wsbs17b3kwgPg0RYfCo8fzAO\nnmEfBRgFNhLc3W6X9vt9cXk8HrPHGD3S14gcugu48b7FYpFt6OM2ixhvVnas9hpzS4i/i0RYfDpy\n7gceaDMRNrE1IeW22+3SbrdL2+3WXWL/cDi4x9XHn+oJslmkPKDmbZtMJmm5XLoN3SbeDYo/T1bw\nsEiExaclJ8Aswii03Pb7fdput2m73abNZtMsrc/bUYQ/OpDFoshRDl7kg7XVatVqZtGj+Nr7egKM\nos/7xd9DIiw+JZEvGMPJ2B2BFi1autbW63XabDbh0vr7/T6ldLukBhRLFtpoUG06naanp6f09PTU\nWOdo/bIIRxYwCrY1RUj8XSTC4tNR447AsDMT4cjiNQFer9fp/f3dXWJ/t9vdTahQdHEwjZfT6bTl\nHsGIDxRgs3QNFGCzgPlvxN9FIizuQt8LOpf1hv0o3tdr+/2+sWDRvcDbrL2/v7eaCS+vmyVcQ9/Q\nr0hwvW2eFYufi4LLljG6Itgd0ed4ZTV/HImw+CvwhZ1LL871ObGC435xH7sb0PL1tpvbAQfv0Mfa\nN0kDs9xqhY2TT+xzR6NRExpn74tW/na7bfmPURzP53NarVYdVwXHHNv5w+QN7/uI2yIRFnelpkBO\nLm2Y+5xcYYKLPlHbbuLk+YF5yQ3D0ex9+9SN8ASsjxCzq8XLZsOoD7OSWYDZVYPWMFrCs9ms8x0j\n/7DE+LZIhMXdiESXL3R0IaBFG1m8tZltXkRELkoCw9Ewjtgb7MqRq3pW8/eeAHugJYzhbPZ6fHpA\nSx5rT1iGnZeSzZXVTJQ1eHdbJMLirni1HXjds3A9UcWY36hxXDDGAXuxwV6ssDUe8PqIAOP+WiE2\nAWY3iJ03u9HkLGC7gXkWsAnwfD7viDAKObtVJMC3RSIs7o4XzYDNhIIFtSbLjZellGMU59y2yCec\nE9BaccoJseeKiF5zOBwad0IkwHYTQR8wDu7N5/NOeBuKbeQXlhjfDomwuBsl8UURRkvXs1C9LDev\nsaDWNM/CZgu8ZAn3FaSSReyJMIvzZDJJh8OhKMD2GvYBz+fz5jyxCNdY9OI2SITFXfCiIVhE2B3B\n8byez5ZFmes6YN8T09y6V1UNS15GlvC1ghQJsTdweblcmvRiq/NwPB5b9R5wEA6/23w+71Rem8/n\nTY0Js/ZZhL1aEhLf2yMRFnfF8wPzbBU48OZltmFYWTSo5g28sbjykrflag/fWoDx770IkpRSSxDZ\nRYADdhzCdzwe02w2a25qs9msI8Dz+Twtl0vXEjbx5aI+eKwS49shERZ3IxqE46VXaMdLqrBsNa+o\nTlRopySubOlyi0K8UrqdEHkWMQtzNLuG7ffqZXCBd6svvFgs0nK5bM6Rl/JslrAJsQbl7odEWNyV\nkhDj47NX48FShq1ZMoUX28txvywsLLa87h0rW/LGrcUIhdizjDlcDPfj9/HqTbALYrlcptVq1XHb\n4PnAympe7LAE+XZIhEU10WOztx4JnVfjAcUTrV6vnkOU8ea14/GYtWgjC7cEWqQ1r/POT+n1jHfu\nLYMOBzgnk0k6Ho+d6msmvpvNJq1Wq2bJsdH7/b7xP+MUSLbNjlMCfDskwqKK3MCat15b3+F0OoWV\ny7w+ux8wEoKL2LDgev5ppJSm22f9o7WFa16DUx7x6ziyAv3tmKZts3BgrDHPxMH1Ky6XSzNvnfg4\nEmFRDT/65h7tvfRiL/LgeDyGdXy9vjfbBYaWedEMfIPwXAyegF679Lb1sbSjJw7vpsGTgeLvZP3L\n5dIK/9tsNmmxWDRia4Jqboblctn4j02krW/HYO4KWcQfRyIsquGBtFJ6cW0sblTj1+tHscFsCXsV\nxjxrmMkNgHnr3t/wttpzm+t7SxZh/htMvGBLeL1et+pN4O+7Wq0akcbpklCAvSQScR0SYVENi3Au\n7rY2G42t2lI/SsKIBpg8C9hzRaTUFtlIfGtfg9vw/NWcY1tGfVt6FrgXVZFSu9jPdrtt5pzDdGf7\nbXe7XTN4x4kq5ieezWa9rHsRIxEW1eRCylgQSxNm8rZIoLlfKtoTWcGl5vl3ufGjf60w9zm/tvTc\nJrkbCL8PvwafOtAFYaALab/fp6enp5YA2/e3SAtZwrdDIiyqMRFG4SsVwsmVj7Rl5LLAdS+NOOpH\nlrB9h5yIRcKLApwTY28fnr+ac1zb+D0j10VKfwbmIhcE3li5ZoadF7OALepElvBtkAiLaqIL1ksv\n9rLdeEoh63uDdTmxzSVdYIse43M+4ZR8IUYxZqEtrfchZ8HXuFeidbSEPReEibTdFFFk0QWxWCw6\nJS/Fx5AIi2pyIhzNVswTZXoTZ3rZa9F6LiKDt+UsQ29bJL7cShYyL73zmDvHLLhYO8LbjpEQ0fuY\nyLIAowtit9s1IhsJsLmEJMK3QyIsqvFEGK1fL86XG06cac1KLZaSKaKwMy8Vmt0OLHylR+kaQWbB\nrRXhHN53QfG1aAcTQYuAsGNmyxlDBjEMjd1K9hvO5/NW2rLFBluyB7op5I64DRJhUU3JEkbx5Ukz\ncw0tr5oBqZrHb+v3pSS+PIOFJ8q83uf8ejcbFGITUXZ1cPYcPx0cj8fwN7TBNmtmdaMFbGnO5peX\nJXw7JMKiGu8CRivKRNjE9e3tLb29vYV9Wx/igs5FQ6Dgei1yU0SCXAtbwNw8cR2NRs2ThIkzv59Z\nvbgNBZhndR6Px01yhoWqWYw2hgDKEr4NEmFRDV7UXmIGR0dwIkWpLGQJE5i+f+slUvDSwq/MKvT6\n2CLr1xPjPpT83GzdemF6WOgdb3DeQJ9Z1yboKaWW0OYiTcRtkAiLavji9ZIzOGQtymTz/LY5UMzs\nkbz272qSKiaTSZOia/V2MW0XlyjCJTG+VoRrlhxHPZlM0n6/bwmwZ4mb8Jr4IpdLu+5HLulF3AaJ\nsOgFuiQ8S5gLrHNK8UcFGLfV/H0uoQIjG2y+NU7V9Zonwjm/cB9q6x9bZps1nuzT8wVHYW+efzmq\nsyzxvT0SYVFNNODDKcqcBfcRd0ROxEpCzNZvLrzMXA82CLVcLpvG6zZdUE0YW1+fcFSPw9tmfvgo\n/dib/ojFlz8fj0HuiL+DRFhUgxdhH0s4V2IyR40VWSPEUbQDrnPR86enp7RardwlztlWckVcI8K5\nKZnwnHMJypTaAsyzMNtnoBB7+3LuiL5PMaKMRFj04hqfcK6uQ0Sfx/gaIY4sVhMxc0fgzBPPz8+t\n9vT0lJ6fn9NisahyRfR1R6AIR2U/seUsYNyHA5olAbbEDs8VIXfEfZAIi2q86Ah2SXjREWzJlS7m\nvn5U+xt+z5wVjLNOmAijO8IE9+XlpWmvr6/p5eUlLZfLKgv4Gp8wV6TLVaozK5tDB6MaEZjMkdKf\nyAlOAvFcEd7gnLgNEmFRDVpRLMAYM5yzhEtW1TUCjH9bI8Q89Y8lJVhcrLkjnp+f0+vra9N+/PiR\nXl9f02q1qnZF9HVHcLGi3BJFFQUYZ8bguOGU2tEReI5MjL00cgnw/ZAIi16wNVyKE7YA/xp3xEcE\nGN/DE4nIFYGTYbJP2CzhHz9+pB8/fqSfP3+mnz9/tkS45Iq4RoRzhfBt+36/Tym1XRCYOMOWMH4G\nYvHBKMpRZITcEfdBIiyqiZI10CdcsoQjd8QtBBjfC1N7c9lwOB28ifBqtWp8wuaG+PnzZ/rnn3/S\nP//8k56fn6ss4Gt8wlGxem+7+W+9+g/sL7b3R+uZk1asXyoJKmv4tkiEvzF4Uda+3huY8yxhLsCe\nixOujYLgY6l5vSeWLMCWBRdZwuaK+PnzZ/q///u/9Pz8fFOfMP4GHNqXW55Op9bNb7PZNEklNZZw\ndA6jELWPivEtb7RfCYnwN8cuoujixP08Qu+tcyyr9xjL/lmGL9bo4i1d1GbhYvqxFanhvomuza9m\nlqR978PhkLbbbXp/f0+Xy6XjU871S98rOjf2PufzuZnlGH8PTKn26lp4VnANOb+/1Ry2GGUb3IsS\nYnCf8JEIi87F7fXN8uX4YF6vSU9mEc49GntL3h+tm9By6rG3bbVapZeXl44Ip5SaFOHdbpfW63Uj\nwmhJ49K+k73umu/hCTGfR65rgULsCd+1QuzFgaMIe9Y/9yXCMRLhb473eBm1WivYE2IDBcZzSbAF\n1WfJ2zAV2VwN0bq5ICwZYzabNW4CtIQtesAr9IPbkRrLkPehAHs3sZRS6xgiS/gaIY4sYXQ3oQjz\nEwAety1FjERYpJS6gfzc52iIkhWcS3dFoTFYhErr3t/wNov7rW3L5bJTHyKlP5Ywuiew/q4JME+A\nmXtE5+/N2zwh5v2RJRx9Fv7WOSH2/P4cgmgizE8F1senABwoFV0kwqIjwCi81mdfb40Y59wRLMA5\n8a3Zz9tSSo0IY90Ha7wdXRU4HTxawimlpo8WNUd8oIh6ff7+3nmwz0afOb8XHmutJYy/eW6ffW/P\nEkafsPmm7SmAfdd9Y6W/IxLhb44nwF4rDcqxGKOQsxVsAsDCdOs2m81aBXgs9MzrYz0IFDQTI1ta\nNtpisXBvNvYd2S9q202QsMwku1NQhL3t1ryBuZxP2PvtvddwKGLOJ5xSaj0F4DkYj8etbcJHIixS\nSrEYY6hSjQXMqcle9AU+ouI6W4K59dLrR6NRJwPOhNcrysN+XMS+p1nDk8nEdbmwUE6n03Q6nRpB\nNdH1UokjIeZtJt4mwizEbAV/VIijIk1mDdv58W62kU9btJEIi6wAc42Ikh/Ytnnia3j+W+/RvbSt\ntG7+XRRfC0Wzvq1Pp1O3boIXLzsa/UnttfOGx2AWKb6GH8lxYDInwLiOTxcYfudZwn0z9fAYagR4\ns9mk5XKZUkoda9eL6hAxEuFvTh8BrrWCTbDYXxstPZHNhTrlxBmX6PfFojzWXl5emv5kMumkBeP3\nwWw1s0i9QThM/rBqZohZvjxYya9J6Y9Q29+gcJ/PZ3dgrq8ljP8H9lp2R3hV8tAd4fn77SbE50l0\nkQiLq4U48g2bJYyimJIfAcF+Tk94S/Gn0X4sS+lVRcP1yWSStttt4/P0fMEmPil1rT/7bA5Zs/fA\n19kSnxbQmvb289IGxHhgjoW47/8B3iS8gTk7B4vFIq3X69bfowDbDUiWcBmJ8DcHL2wOR+NH0pL1\ni/tS+uP79MTYE9lIfEv7o9fnLGEsTfn6+toIqB3z8fhnVgqME16v1+4AlFnAJox4Xux1tmRrmOGb\nVvS71VjC1/w/1IaozWazzlMJPglE0TGijUT4m8PCiAM/GGx/uVya5AcUNa9usDdIVRLOjwhv1F8s\nFo3oWiYchp+l1J5SCIWGB6Ds8ZsTFFB45/N5p1iRCVokvp5Aefu4fz6fO8V9omiNvv8PLKrRd7Vk\nFx4gxBtCX7fId0QiLDoXHsenGlxpDAv14GM3W8B9hTcnyCU/MC7n83kzKGfhaFZhzAQRox6w+BBO\n02QCjFli6HZgAWYx9PzAkXWIlii7idhKNR9tn3Khpf8D73dD6xZno8ZEl1zMsgQ4j0RYuCLsXbxY\n+Bwvfn48NwH/qAXs7YsG5ry+xfOyWGBhHXvkRlFjAfYsYRalaEJTtILx3OZEOIrV5gGzaEZrTh6p\n/R/gdU+A8UZszUTZC5eTFVxGIvzNYevHC6ey/Vjq0S56T4AtIqCP8NaKMYst30BwPaqaZum0aAnb\n4BsLsSfC5o9Fi9Cb1NTE0M4r9iN3BFu7+D7YPx6P2clU0Xqu/R/AdYz24HKfUS0OPL+yhOuRCIuO\nkHn7LCvKLj684O11Zn1a8ZucT7ivGPN71TT2aXIEQUp/LGELS2MB9twRLMLsm2WXgFnDngB7sAB7\nZULxeK+dTNV+N2+b545gS9jSvz2/MNexEDES4W+OJ1zRPnRHYPwnC/B8Pu+I8K38wnhRe8Ibbfea\nWcLn87mTlluyhE2IvElNPZ9w5IqIws+8VHFuPIOJba8dmMuJI9/EWITRJYElQT2fcOmzvjsSYdES\nTHtkHo1GHX+mFcRBK8sT4OVymbWCP2IVe2Ib9SPR8wa8MBmhxh2B1mDkl2V3ROSSQDxXhIWI4RIH\nEaPPzQlxSYA9nzD+xuxn59Ke/JuJGImwSCl1s7QwcN/E2RIQsFaCN0nmarW6akCu1hVhx4t9b4nh\nZ15DoYsEOHJHLBaLZj9boyyE2DyLOOcTRssXxR6Pt3YeP/6tc/8LeM7REsZi+Fj2s09xedFGIvzN\nwQuErceU/oiCCTG6IPDixDoNu93uw9ZutI2FFvu8jdONrdn3wkEuLzIiZwkvl8vipKacrJCLirBj\nsqVnCXvWOn62ZwXzZ9UKIgqwF6KGljDP2edVdBMxEmHhCnFK3bnmMA6YXRAsdCiat7CA0b/Ixxxt\nM7+pNY6KsCXXy43cEWgJr1arVppzzhrNibGXqIHuCM5Y4xsFuyOiKnbROYv+H/DceyFq5nZaLBYt\nsba+N4gqfCTCIqXkW5bR6+xCm8/nYT2JnE/4I9tqjt/Y7/dps9m0wtLM8rX3wkG5nDuCLeHNZtPK\nGKzxzUYC7N3sOBW8ZAVHrpCa39Q7l6WBObSEczdSWcJlJMLfmD4XBw7C2cVp2+29bD+KcCTG1/b7\nHLM324RnofWJrjCfrvce0fvaeTLrlgcObft4PO4Iv9e35dvbW1qv142/GjMYOYa75rfH7x8Jqudu\niH4vWcB1SIRFbzxRuly6U/GweOaElZeeMN7qmHMCi8ISxRjbd80JEosRxgtblTmsWWGWOrs/cu3t\n7S29v7+n9XrdbPP8w3weuM/bIgH2BLbm95IQ55EIi2oi8Y1ewxdpTf+WwovHFH2HkrXHj+Q5EY7E\n2DBB5PhhPJ/sf/Z80p4Ibzabjo+4ZAnzebZljfB6+6LfUeSRCIteeELM+20ZPZ7XPsrj533kePm4\nvc+uffy2iSyjGr5sTbPrAd03fIwmwhgOh31evr+/Z0WYLWHvXHjrNcJbEuBb/HbfBYmw6A1fnNG+\nW7RbHS8vS4NJOUuXLWF8vfdeKL5onfLg3Pl8boSXhdjbvl6vm8buCLSEzZ/Pv0/p9yq5JWr866KM\nRFhUUyu+ti+6MHMX7b0u6JzIsOCwsHqWsBcXGwkUCrBXEc3WT6dTI66e+PI+towxXM6b1SM6J7VW\nb9Tu+bt9ByTCohcsmHYBo4/T+t7ro8fV3Ots20eOlbdFosOxrizAkU/Ys4bxM3JV0bCZyJaWGBGB\nIXVsCXsDc/zd+wpwzU009xuINhJh0Ru7qPBR2yw+C9/C13nL3D5v+dHjveZxG8XVBDjyCUeWsN2U\noqI8nJBh7gV2N3jbcBYQjCPmWU74d/POCft3S/5f3n6v3+47IBEW1fBFnCuO41mxkWXb57UfOfZa\nCzDnDy5FR0QCxSLM0xKZiJrA2oBbrvHfe0WEPHdE7qbE56LmpnXv3+6rIxEWvfAe71PqlmSMXv/R\nbdfguTlYWCKR8dwRKaVQfKMkhpRSR4QxA86y4DwRtj5v4/rCvMy5I7DPVm6fp4XI3SDxrUciLKow\ny9f6uPybx3Dt3+ASRbdGfEshapH4siVs1jBPIY8+XRNYDD+L+ibq6G/Gwb7cwFxf10xpYE5cj0RY\nVNNHBKO6CLn93nq0rebYTOwwUgDdJSY+nkXLFh6KqFf4Z7fbpc1m05n40uaCq2kmwugD5gE4rBUR\nRVlwPPJoNApdJ972l5eX9Pz8nJ6enjqzZ9jNR37f2yERFnejVLQmKmQTxdEykWvE+ugr9SbARBHO\nTcnDRXVsUlATTk+g7G82m407izNvQyFH8cWaEN738LLi8AZjNYztO3LxdW/958+f6fX1tSPG9j1N\nwCXAt0EiLO4CCiwLBm/j19dUG8sN5qEP1pv3Df9uMpmk8/kc+nHZ0sR6D/v9Pm23W9dCNNHGmalL\nzSudGc2ewecz8sVbBMtkMmlVQeMl9n/+/Jl+/PjRiPBqtWosYpxJWdwGibC4Gyxg3uOzJ9As1J7Q\nROFu2MeZKbwZJyJ3BFt4eMzohtjtdo0A299hFMThcGiJsAk3h5Phkt0T7IYoFW3Hc2N9TC7xpqvH\n9cVikX78+NERYXZJSIRvh0RY3AUW09IgEotxtDRwUInX2RpFCxaFC61EtPCiATWs77vf79NsNkvb\n7bYTAYGzdSwWi5bQ5lrOQo5qQvBTgomunQfr86wYy+WyaWbpWvvx40fjjnh6emr2yx1xHyTC4m54\nAhzN9VaylrHyWEp1NSr4JoBiziFZpUG5yBKOQtBMPHHWEZ6wM9qWE22vMA/juWrYEl6tVk0zobWl\nCfDr62vWJyxug0RY3IXIEuZ4Vk+Io0kyT6dTSillU2fZGsZj4b6JcEqptyV8OBw6r2OBNn8xZsTl\nprD39ntTF9n5iZIivAFL8wmjJfz09JSenp7S8/Nz056enho3hDX0CbP7RXwcibC4G9GgljcdElvL\n0TKlrgjnkg2MaCDPRDjyCfPNBC1dFmCzVne7XVosFmmz2aTpdNpJT871vRsUb+PqaOyC4O9oIoyT\nspob4uXlpdNMnNE6liV8PyTC4m5EVqRn4XmuCm9bSn9E2BPf3L6caOfihL2bCLsg0KdrUQgmWDlx\n5X25JwJez91kuKFP2ATYrGBzPZgvmH3EGCvMUSDi40iExV1gV0QkxBg+FokUbk+pLcJR9hv2Oa3Y\nrEYMSbNHbBZg/h527Cl105Cn02nabretmNvxeNy5oeT6XlSIF9qH38GIhBjjhNkdYSJsERE/f/5s\niS5HUcgdcXskwuJulCxhr+CM91iOy5SSK7TRtul02sQBT6fTziN7FKLmDczhcdl7Ho9Ht/ylLS1U\nruTrtj6fv1zfojvsuxgswHYs6BP2LOGfP3+mf/75pxWO5jW5I26LRFjchWhQDgXYK72YG7gqiTBb\nwlyCMre0bDUvNtdzFfD3NIvXuylEkR5eFIhnXUbb2NLl2he4zrHAHKKGzZIyOJsuF0strkciLO6C\nN5jlFa7BUCz2F3vbUoqL8HhuCa/uL6cpTyaTtN/v0//+97/077//pt+/f7dmMcbZKtgi9bL6MBMQ\nRTh6bU3CBfftO2JUR678JoqsuRvM4sWMOU5h9jIJvYFPcT0SYXE32BfMFrBncZYSGlJqi3CuXypU\ng9sPh0P6999/GxF+e3trLOP9ft+pz2s+WU9Yz+dzU7fBRLiUARgRRTyklDo3FBRO7mOUA4qw53pg\nyzcXQy0+jkRY3IVaS9hEDi1iTufFvpdokYt8YB9t5L89HA7p9+/fTUNLGFOG2RJGIU4pdQTY9peE\nOJeSzevWRxH23AfWn81mLSuYIx7QEvZcD+zekBDfFomwuAteZAT7g7E+AtdUiBqKcKlF/lFv2/F4\nTO/v7+nt7a1ZlkTYvicuU0qNANvAWVQLw/4uckd4AhxFPUSDaNYid0RkCXs3LLkh7oNEWNwNS2LI\nCTAPhHklHrGPFqbnn4yE2Gu4/3Q6deZxw4E6dEewaJq1y2B9h6jVEFn+nhWMli1WRrPUY2z8WosD\nns1m4Q2LXSLi40iExV3o447gWYOxcWUxr34E9nm9ZgBvNBql0+lULCXpWcL4XaMbQzQYF4lxZPVG\nVj4LMMf2zufzTvabRUigSwIt4VIYoLgdEmFxFzg8Da1hrp+72Ww65RtRmHEbi3BpWfIbWzufz1k3\niDdnW86SNVHG9ZwbouSS8G4q6BM2AeUwNOt7lrAnwBYHnLthySVxWyTC4m6wJRy5I3KNX4ciiELg\n9XP+VM9aLYXHRdER/J1xQA6355Z9jh1F0XNFcBwwxwJ7PmFvdpDSORO3QSIs7kIfS9isYW/J27wS\njpEoRG4Lb5lSqk4rxu+YE2LvnJT6fPyRBR9ZwuiO8BIxaixhjoDInUfxcSTC4i5EPmHzr6KrAedV\n48b7rX5EH1gwvPVc4kXOfxsJaGnQLbefrc1ooJFD1NgS9moGR3HCbAlHxyVuj0RY3A0Oz2LL2EtT\njmabMNG+RoRLoEviI5Qy3xCv9gJauDVLE1cuOxktMS2ZQ9M4MUP8PSTC4lOAj8EfFUvvff/Ge3vR\nG7zPq2thWee1AAAgAElEQVQR1bxgKzfX2BJmy1flKYdDIiweipwQ3FqIvc+6x3vnojJyA225DDgU\nYRNXb8nb0BfMVdEkwsMgERafko+KZUnsb/neLLS5Pk9HH/WtOHs0WWfU0BeMlrB9vvj7SITFp+Va\nsayx+G753tGgmpdKjQNlpT4Ka6mPf8fT16tI+7BIhMWnpq9Y9hGaW7w3ulByxYSsmbh6CRdR31vn\n7RhHjBY1WsKKAR4GibB4OPoKQa1YXiMwt3hvtoSj2sbT6bQRT3QvRH1OT+a+Ny0RLj0RFn8fibD4\nEpTE8iMC89H35kE4ju/1ZkK2OeByERCRu8LbFtUZxpuBRHgYJMLiIbml1XoLcfnIe7M/GKMgsHmJ\nFs/Pz00ssM0JZ312L3jr1o9qKXsV0sTfRSIsvhQslrcUlmvem33CniWMKcMowia2Ly8v6fn5udOi\nCApvPReVgdvE30ciLB6WawXUxPIelt017+25IqK6DzYwh7Mh44zILy8vTYuKuXvb8TisH20TfxeJ\nsLgbUUKC90jszf/m7UdKpSSvpfS31wwcRvV/ufoZJ1qwGwIt4VxCB/clsI+LRFjcBc/68yy+w+EQ\nTv8e1ZtIqSuUNRXKvPKRXh8FKyri3tcS9gTYCyczgeWqZ169h9yEnHItfB4kwuIueNYfC5AV6eFJ\nMFF4WYBZhHPCGhVRx8JC0YAbCnL03rVCjCnJ+P29jDYceItSjS3aIZqUk9OixWMjERZ3gy1Ar+4t\nzljhia9XdS2lWGhxW22zv8Fqaii+OaGuCU/zav+iEGP4GYswVz3LVT7TjMifE4mwuAsld4SVrjSh\nLbkgrE0mk2LdX9yGMxzn+gYKcE2iRh8h9gQYfb/mhkABjtwROTeEpiH6XEiExV0ouSNwtopIfL1t\nx+OxaNGy0Np7jMfj1jxxto+FyhPinCB7vmTvXNh5YBHGMLSo/KTnE2brFzPfFPf7eZAIi7uRiwrI\niTAKLxd9H4/HRfcCW7nY0Drk9ZxLoobIKh6NRq3vj1XQOBTNK0PpTUWEMyJ7TZbw50EiLO5CJMDs\ngijNusHrLMIouJEFfDqdsoLEA28oxPZdasXYE2I+DxgZwUkZufKTnIDBiRdoAUt8Pw8SYXEXTAi8\n7DAU4JRS6ANGETZLeDKZdKIpWHhxmwmwLRGMkODt+D36xhzze6I7gjPjbGDOLOFcpTT2CUdF4mUF\nfy4kwuJuRJEBPGtxNAhnIszuiCikzduGgsTz07H17NFXgPHv8LO9CTm5RsTLy0u2KhrXh4iy3ry+\neFwkwuIueO4ItoDtdZEVzCJ8OBw6Iuy5NdgK5uQLFN57ChW6MzxLGN0RWA8iVxkNLWG2tvn8i8+B\nRFjcBfRRcpgaipNZqDhQFw3YXS6XdDgcsgKM29iaZnGfTCatfSXXhheJ0edcROeEY6i9TDgvnVt8\nDSTC4m54ooOCZqAAo5/We4y35I6SG8JE2LOqo218A4j6eJwp5eOUU2q7Ww6HQ9rv92m326XtdpvW\n63Vj3Y7H48ZC5sFLO58cZhdFY+TWxWMhERZ3gwXYBGU6nTb7U0quAHPdXcywiyxWb1CuJLy4zbOc\nvSgNs5pLn2/Yun2WifBms2kVXR+Px+lwOLRuCnhDYtcOxjDjOee45ZqEEjEcEmFxF6LHb7QQbT9a\nkSzA7ENlAYzcBWYt1lrB5nNG/3PUr4lv5u+Fn4GWMM56MRqNWpY+W8DT6bTjOvGE2P5G4vs5kAiL\nu+EJMO5jEc4J8G63S8vlsuMOiBqKcI0VbMWErO33e7c/mUw67gvsp9QtKoTijCLMSRcptZ8KUurG\nGM9ms+Y1eB75vEuAPw8SYXEX2BK+XC6twST0+dp6JMD7/b5pJQFOKXWsz1or2D5jt9u1+rPZrFk3\nlwEnkFhhIfx8XOfP2W63nUk22Y3BLghLdsHwO/yMXLSEeFwkwuJuoAh722yQiQX4cDi0Cv2YFcqP\n6tHgmK338QkfDoe02+0aN4H1Z7NZ2u12LZeBie7xeHRD4PB72Tazyk2E8f284+dEl9ls1vEX1yZm\nSJAfG4mwuAueAOP28/ncWMgowMfjsZXejIka+BgeRSLgso9P2KzTzWaTFotF08dwMfsuJpCcIuzF\nH6MrAkWYp5lHKziqt2E+cTsPXuF2Ce7nQyIs7gYKMfY5msAE2IsX5vVIcA1PhGt8wvv9Pq3X67RY\nLNJms+nE65rg2WP/fr8PBZhntsBoDYuOYBeEfU/PBTGfzxu/NIfo8Y2O/e7i8ZEIi7tgAoAFd0x0\nuAgPZ9N5iRdeth32ed0T4ZwY73a7VnowCzAOINr38wTYEkB4P1rCOCDJURyRBWxTQbE7wm5qeJOw\n49Pg3OdAIizuBtYwiKxVW3q+3cjfy+/jwdanJ7y4z0TYEicwQ40F0xN7+yysamag0KLVitv3+31T\noAjrS+DAJLtlUIDxXEmAPxcS4YHICchHeJQLDwV4KKIEDG/pRSt4NSc4gYJjga3hwGNKbcG192Vf\n8el06kwCaoOE6I7AWGEUdX7qsON+lP8J4SMRfiA8YcbHS0YXVx68EfCjO4bMXS6XTrF19L3ae5ir\nYD6fp81mk7bbbaveAw+2pZRaNSK8wTgTb3stxiZjmJxFbWDDueW4b8etGhOPj0T4AcgNMPF6JLwS\nZB8O48JHd+R8PjdxyZiabO/BRdm5olkULcEJK1FEhL3WE2EUYhRjOx4MZcPvFn1X8VhIhAem5B9F\ncimqIiYSYOxjNhpnpHEZShRfzwJG1wW+B8f0onvDjgUz9Fh8TYCXy2UTwXE6nZrj4EHDKIxNPBYS\n4QEpDUzha0oCLN+fj2cJe/svl0uaz+duyjAXEVosFi0RjuKF2ZrGz7TXsc8YBdhzR1gInc3Tx0Xy\nvVBA8dhIhAcmFxkQhUR5SIBjPPHFfRj94LkguIYFFt3BrLeU2nUi2K/s3XRZ9CMBZmsYy11ybLBq\nDn8uJMIPQiTAXKiFH2klvjEosJ4lzNvYn8rxuiaQi8XCHYSLwuI4vI1LXmLERc4nbOJr/uhIgO3m\nIEv4cyARHphIeHlblKLKF6Hogokj5idFcY6EDIvmYEnL5XLpCjAXkccMNy5cb3+DlnNKXZ+wNyhn\nIuy5IMx9wr5m8bhIhAeEB4k8K4ldEl6KqvzCMXw+OFsPBZoFGON+sbD7brdzB9jwtSimlkFnYEQE\nWs4ppY4lHLkj5vN58/2i45Yl/DmQCD8Ank8YH11TSp0MKXu9sqPy8ICYN+hpTxp2fktF23e7XfN3\nUYEe8x1bJTh7PbpGWIQvl0sxRM1ik1GEeb46FGCJ8OMjER6YyBeMF3hK7UdpFm0JcZ4oMsEoDYxy\n2+12rd8oqkm83W6bQj/4e3pRFOa2iAR4Pp+3kkNsyntMBMGZN/B/Rzw2EuEbU2t5RILrNXs9Pkqz\nj9j24YVXEuWvLtqe+N6C3W7XGYBjAV4ul2m5XKb9fp9SarseuH4Euz2irDlMEpnP56EAyxL+XEiE\nb0guFImXfPFx48EczLgqNUwO4CX3xXWgFcoFd1arVVPvwYryeJlyp9PJ3WdL7+bs+ai9G7cE+PMg\nEb4xNY+zXL6QC5fztpwI8zauIeCtXy4XxZB+ALyZWcIGTka6XC7TarVqBuV4Bg2vfCVHWnj/L3yj\nxsiLSIAlxI+PRPjGePGf3tImfcSGj6LYPBH2+rZEYbA+prZalpe4Hjz3mM7MtX8jH7D9/l68sb3W\ns3A5BjkSYFnCnweJ8A2p8e+iFYQ+P29ABkOcPLH1liYKVt/AlrPZrBXSJkv4eswS9twRKMAW8WDw\nIB7OtMxCHVnCkRsrEmLx+EiE70B04eASJ5Y0/yHHg9o28yvmShdylS97NMaKYCgeukA/BteVQEsY\nIxRQSNHFhJN98lMJuhIiAWZLGD+PE0LEYyMRvjGRBYMXjT2K4sy+XCsWt0Ui7Pl9p9NpdiocpbPe\nBrSEsbYE/s74v4BuhP1+n61BnJLvjigN5MoS/pxIhG9INJDizeSAsaQ2s681XkcRjsTXBNimjEfL\niFNasVSj6I+XpWaWsJepxqFnu93OnULJqBmY46mOJMKfF4nwjckJMT+OmghvNpu0Xq/Ter1u9W3d\npkjHyl3cbJ/VmEUhQLHgyl7iOrwyl1GSBP/mXAzeK4OZC1Hjpyr8XBZg/c6Pj0T4xpREGAPxWYTf\n39+bJfYjEeZZFWzqHS5KzsVosM6t6A8PzHk3PozHNgHGKmhRQXiDxdgLT0PxzYWpicdGInwH+MJh\nAc6J8Pv7e3p7e0tvb29N3xNh6/PSqmtxPVwrxShL+DZ47gj73VGkR6NRx/VUmpcuF2VT8gkrRO3z\nIRG+ITVZTjkRXq/X6e3tLf3+/btZ/v79O+12u5blhEvetlgsWiLA4VM4aCSuxysbmVJXnMfjcRPt\nstlsGkvYRNgLUbMl/y9hJAS7JDgyQu6Iz4NEuED0T+xtxzAkrn5lfVtuNpv09vbW+H1xanN81EzJ\nt7rQmsL1xWKRnp+f0/Pzc1qtVmm1WrUufG82CNFPrDjhwgbbvN97s9k0riX7nTkGvM+TiUT16yER\n7gFeAF4fL0x+BPVC0eziXK/XzX4MLfMG1riIC7flcpleXl7Sy8tLI8TL5bI1L5rngxQ+nuhh3C/G\ne3u/9Xq9bp5sWIgj95CE9nshEa6EH++8JV6YGAfshZ1xOFruwsQ6BZwea0kZtr5cLhtL+OnpqRFh\nb0RexOREkTPfbNDN+43NxYTWsFnKZgnLPfS9kQhXwH66qI++X/b5eqFnnKwRWcI8Em8WrxWKsSU3\nE2GeIViWcB5PgHkb3nDR9YC/sQ22ogiz28luuLm4bQn010YiXIknul5MJ1+YZiFh9MP7+3sT/xvV\nikCfMPuDTVRNaE1s0f3ADX3CEuGYSHRxyYOtfMPF35l/c/t/uNYnLL4eEuGeRALMqamehYThZ5vN\nJqyehu4InAMN3RFm/T49PTXuB3NBmEhjDQm0hL16BcInevJhdwT6gO23Rl8wPwGh68lzR0iUvw8S\n4Uoiy5fXvYE5FuHfv3836cgcbsS1B1Lyp19nEX55eUmvr6/p6empU0ENK6nJHVEmEl5bYiU8vtma\nJYyhhjwGwD5hJVd8byTCFXhuCK6QlbOOWIR//fqVttttOCsCBt6n1B2YY3fE8/Nzen19TT9+/EjP\nz8+dGsJcX1gDcz656JfI/x9ZwijCHBnj+YQlwN8XiXAlkT8Yg+Rx1LzGEub389ZT6qYe48Dc09NT\nE5L2+vqaXl5eshXXODlAxES/jWcJY3QE/s6/f/9uxYzjMueOEN8HiXCBaIAmJ8CYFccibBfmZrNp\nhBALuHhLL044soRfXl6K88xJhGO8myBvj556vPTz379/t/4feNYUDcwJiXAF3qMpW0WRT9gbmDMR\nLk3UyZNzojvC8wn/+PEjvb6+ut9BotufmkHYyPVkv/OvX7/CuQM9d4TE+PshEa4ABTY3YwamrHLY\nGVtCNsEjTjM0Go3S5XLpTNzJFbdKTfxHLuHCu7FGBXN423a7Tb9+/Wp8vpj56KWgewOwUbEdfjrC\nG7I3pRX28W/4iUo8LhLhAnyBRiUFTYSj2F+cd8zadDptXWgpdQuGRwXc2XLWxRbj+XWtb0sujBMt\nT6dT2mw26X//+1/6999/W2KcE2Cv0lkUFZETYLwpRzdmFmLx2EiEK/D8vt4jJs8P582mjBclXmgc\nCeFZwRLg/uRcCuxWys1+jf3tdtsI8K9fv9y0ZPT55qYhQtiSZXGNCvpH/xP6v/gcSIQr4LAkFF68\nQD13RE6Ix+NxxxriC7D2YtNF51MT2325XDqzX+faZrNpBljNEi6lJbP7ynNFGJ4Ye1NZ8ROSN/Cq\n/4nHRyJcIHJH2EWLAhv5hPGCZBFGK9hgdwRbw7J6+hENonrhZlgRLepbGVIsvO+JMEY/1M5+kbOG\nI3eE/ic+NxLhCtBnyGFoNuDmxYGySLMbw2Y+9hIzcpZwZPWINrnkGm5R0SWugMepybi017Al7H1m\nzhJOKbX+B+SO+NpIhAt4ljDXh8BwNBbgnBCfTqc0nU47iRkcG1zjE9YFl4d/Q46AwN8ShdZrFoYW\nlSblAj1seXuDcpfLpWMFs1tK7oiviUS4AhTiaMJObqXICC9UyfDcEVHWm8KR8kRx3fhkw+4IrAGB\niRc4AWuuWD+XJPUs3xqfMMeMyxL+mkiEC+DFy6FpkRCXBuaikKWU6twRuuD6kRPgqCQlJ9dgGvJ6\nvXbdT1FaMofH5dwQNRESClH7WkiEK+jjjmD/cGQNsxAbeAFFA3MKU6snsoKjm2okwhaO9uvXr/T+\n/t65uUauJ6wLUVrib1iKksnVBJE74nMhEa4gNzCXc0dE4Wk8o0LOHSF/8PV4FqiX8ZhLPzYB/vff\nf5v2/v7uupe8lOTT6RQeV0RNnDD/X+TcVOKxkQhXkntE5DTSKI43ShKIGl/gXgF4DoWbTqfN8fLx\nR+t9+zV4QlPaxpZhaX9uyT78qNm53Ww2rQSMKCXZBt+irEn29fP5Yx8+9ieTiTsrik1X5a3jTNqe\ni0JC/PhIhCuIguaxTu/pdOpcCNFjol0YuRjkw+GQptNpR2h3u11aLBZpt9s1FdW2221T4tJ7FI0u\netzmreOI/bVEQsnbIn9pbjDLG/DCbew68pJsrGEChjWcIRnDznJZcLknG2+wDbdNp9NmdpSaZqKM\nM6ng/5+s4c+BRLiCyD+HAmwiXBJivChQPKJsvMlk0rF2UYCtsQhHwlpq0ePsNRezJ6ZR36upEG3j\nvieCHMnipSDjcrvdthIwMAvOm5KIoyv4hmDfzc6dF2rGT1Cz2axagG0aK7aGbR5BDFkTj41EuID9\nE3sj1FaEBy3hkggbuRF7FI/x+L8SlpEI22fOZrO03W47Ilq7jtl7uA3PwbWUwrNqRZVjfFEEvSWH\nE3rN9ueSMDD21xtUjZIw8H/I8/Fzv68I4xyC9v9g/3tyR3weJMIVeBELduHgxRhZI5F1GQkxCvB4\nPHYFeLfbtQTYWvSoGz0Go/iaAKeUmm3XXsR93Af8/T2x9W5UuYp2UWp5VFwd44O95lnCuZtGJMKT\nyaR1o8b+fD6vFuDVatV5EkIjQJbw50EiXEHOHYEXH15QnjWcE2EUjslkko7HY+vzIgHmCzAaOOQB\nQ29AcTKZdAbPzDd8LTUuBvavsrhyUkUpIsH2oeCWllwnghMx2CfsWemRO8KL8eXfzYr1W5H+GiH2\nbsLe/5x4bCTCBaKICBbgy+VSdEegENsFGrkj8LVoDXsWMFpVPICYi+Aw0bU+fmdbflSAbZnz67Il\nixYt9z0/b7TkQU0vmsTb5hVh8hIwvBtKjTsCb6DoSjARrm3RhK5oCYvHRyJcgWcJn8/n5qKyi84T\n4VxCRS46gl0gnijw51mReLTWo8EgLB5kYW38nT1B6UvJ/cD+2xort9bF4FWzi6rcmZWbqymMERW5\niI7ofwdFGP251mzi1tpWSt5QdMTnQCJcAV9IKF544XlWcJ/oiPF43BHglFIzco7iy/5EHJBhseVm\nxx+5H0ajUTqfz41f+Foiv3BOgHOhZLXWLboYoup2uZmPS9Y4xgCXoj5qRBjjf/uIcC5GXQNznweJ\ncAEUQhZiFij089WGqHmixLHEo9GoEeHZbJZ2u11RhHPZVehKyYmwCfU1eK6IKNLBE+LIEo1E1Fty\nP7d+OByKkRnY+Hvm+izCKMSYfNFXhL1ol1yYoXhMJMIVeO4IT7w894AJX84KZgHm19h7m/jaRYzr\nGKBvfdtuJTNRgNGK5ws459/sAwtxJMDWci4GHkDjhoXX++yzxoV28Lh5W1+86Ai2hFmAawbn7L3x\nc7y+eGwkwhWwCONgFl6UuYESHkyzKAgexeaIiZT+s8JNiHIhSObWQOu3tETXBrs5eFsfStEOuL1G\nfDGUzKvTwULr1fTIWcVYaKf2fwL/N3jd+pPJpCO2keVrwospyjhwx+nJ4mugX7ISFGIT4cvl0npc\n9wQXLyAchDGfa+TDMyFOKbUeyXe7Xeu1LNr7/T4s8OK5J0rCi8s+eMkUOREuJVV4/uCoap1XThT9\nyxxihr9xrRBHMdccSTOdTqvdCyzCmAmnsLOvi0S4APvb8NGSX8fChenELMTn89l9bxZhc0eYAHGY\nm4mJ+VMXi0U4KOdtjwTXE+U+sPDmhDiXUszbeDCu1Mell+12TVy0FzPO59bW+2bB5YrzSIS/JhLh\nSiJL2PaZ3zWyhDkUif2/DPogMTQLhRr9qyZY8/k8G5rGyxor+BoR9oQ3EuKaAjt9rOVoYM/eK5di\nbL9njRDjkwX65bHfJwsuckXY+VfEw9dEIlyBJ8C8zyzhkkvCBPl4PKaUynUVzNo9HA6ti48f580C\ntEdXz9XhJWvUWsG38gl766Vyk1Hj8LUaMS9Zwvi75vbZ/0PufNlv30eEvcposoS/NhLhCtgV4W1H\nEY5cEmgNY7ypFxaV0h9L19wRhg3AYX0ETGf2fJNe2nIfEb6lT9iziDlbzkvaKO0vvZ5jfkuRHzkh\nRvcDpiHzIFrfLDj8H+HBOC/KRnx+JMIF7EJkEbZYWi8Yv2ZwDgeITBRSarshMELCtpkAm9B44Woo\ntKXlIwzMeWLMfW+9b+NjylnC/PvzNrSE7RyheKKY4sBbLhZ4tVp1oiFkCX99JMIVoBCn1K4wZkJ8\nuVzcULQoOgKtNx6Mw0E3tNhsm4nv4XDohJ1xYkgpoP9eA3Oe6NWIcM1rMM446uM273Wc9Vbz+yNe\n3K/9xuZOMNeCF/0QibBX2Eci/LWRCFeCsZ8oiiig7BNmVwRbwl6kAwsyCwf7d9HP69UMiCIwrNWI\n8LU+4Rp/cK4fbWM3Drt0vH25Ze3vz8ktkSVs8cCchIEJGJ4Qr1ar1jln95aK8nxNJMIVsADjBYnL\nWlfEcrlsWcAp/Xl890T4dPqvroTtR6FlFwMLsC2jbff2CfcR3pxlyyJ7i4a/X+3/gb0+soQxIYMF\nlrPgvNfkUs1lCX9NJMKVsIil1L2A+yRreANtXrKG+UKjY4j6teuPIMIssrmG5x3Pf00/t60Wdumw\nJYwuiD7zxVlkRFSQh2uQiK+DRLhA7h+e95X8hLvdLq1Wq3Q4HJpsuyhl2dKaeT++zuB+7UVqInw6\n/RfjbL5m9DnfMjoicjP0dSv0hW9UpZtYtLT+bDbLFt15fn5uNXwdz5zMA3G5DDx+0hFfA4nwDclZ\nRzgrgwklBvWzfxgjIVh8S2LcR4jt9ewO4Au+r/iV3AuRAPdxG9R8xxqXjPWjKBIvogTFFq1eb1vN\nNPUsuuy7r/2+4vMhEb4xGKqGVjCGpKX0xwr1HjM5FC0nUJGFWCvE/D7sm8bX9KEUmRD5efl72ucj\npe/lWb41zRvk9JYowlHVM68Yj5eEgXUhIgGWEH9tJMI3xC4ib7CGBTjy87EvmNNsveV4PL6pEI9G\nIzc+uQ+ee8ET3miwre/nRv7vUrgeWp84GIZhf7zsUw8CY395Jo1aS9i+jwT4ayIRviEoruiOMAHG\nECfOYEupGxFhabdoRY5Go84AlW3H0XsjJ8Se+Ob213x/I7pxlMS3JqIh+kzPRx+5GXgbFjPKDVZO\np9O0WCxavmD09/K25XLZSb7wkjDYJeVZwdH3FJ8bifANYRGez+ehBWyPn4ZnAVtNCB7EMncB+m5z\n9BFib9/53H+GDXYteH7fnAj3JRpEK6Vwo/uBY7txiX0UYR6g4/XlcukmX2DznohybhPxtZAI3xi8\noNECRr+jWVQlC9gueospjny1txJiFmFzdZi7ow8l/3VuAC4aiGS8gTZeogDnQr/Qh+8teRsXZ4/a\nYrHoWNdew7oQnhtC4vt1kQjfEM8SNlCc7WLkATEsWYlV0ayCGvpqWcBqfb+Ry4F9y/Za2x69f42F\nXRJcHoDzBuSuEWIW4dyAGxdZwkE0TEPmiTk58oG3LRaLsKC+V+85+h65cy0+NxLhG4IDPCgaaAHb\nxT6dTlvuhdPpT01gb1p7toAnk0noy80RCXFKXd+y9dGd0oeS4JbENueW8HzBngXpWcGe+KEIo5jW\n9r11E+0ovdxb5+/jfUfxtZAI3xi7qHAQzsQXi/bMZrOWCPMURjxzM2IChq6CPo+sKMQohDnrq+a9\n+UbhLUv7+DhriR7hUYxzUz15Iuz5erHmb66h5exFYkTbovMp8f26SIRvCPp9U/ojyF72GIuyWcC7\n3S4tl8u03W5bI+gGuw+u9RdGQozfpeb71nxOzbZriCIHIlcECjGHnnkiHMUAW91fdk94rov5fF7l\nu8al+F5IhG+IXfS4bo/2JsZYA2G1WjWz/WKfJ6o8n8+NlTyZTNLhcOiksuJnXkONX5Y/g61n7xhu\nKSz2XrWxvzz9UNQ3/30u6437HPMbNRwXEMJDInwnolFuE2mOJbYRd0xvNrG2Og7RJJbYt2I/ueNC\nOFohF0LmPeZHLfq80vbS6yLrNgo7y4kv9m0aIo759Wo+cMYbvp9XTlSIHBLhO8ADW7bN/LcmEjhQ\nt1wum/nQ0Fo2n3I01TtuM7HmY8mRSyu2Qbko0STq89NAqd/neKOBtSjioCS+GMlSE/FgIswxxHgs\nuWgSIRiJ8J3gx3QUYHNPTCaTxg+JU7Kb6Jm42PxxXkMB9kS4RG5mC/YZs9Dmai3cyw9asm5rLV/u\nR7NieH5eE2HPEvYyIYXIIRG+MZ5wGSbAJrBsCWPFNPRpzufztN1u0263ay1tOwpA35KT3mSZJiBm\nleN3y0UaYPPcMfge3K89t+jGqamBXCPEdo69GVCixI0opRm/uxA1SITvROkijOpL2N+iSC8Wi7Td\nbtN2u02LxSJtNpvQCtvv99XHcrlcmsgM9mWiX5hdKjX+1lLIGG7vc+5QaEvpxSUh9gbnsPASv3/k\nB/YsYQmxqEUifCcicfEs4ai+BIZNbTabzoCQNwFkH0v4crk0Arzf71vHzGFwdmxsCbMlaP0+g3h9\nxJKxU8gAAAT+SURBVIrTi3nmEi6QUxJf7HvWdGRh57Lg5IoQfZAI3xgvfMu22zbzCdvFjQLMFrLF\nDJcsYAzLqsWOI7KAJ5NJZy48zxr2rNNcCBlv7wNGk5Qa3qRq3RI1+3lA0PORyxIWtUiE7wRHSKAA\n23I6nTYC7EVMYAYdz8YQhUNhZELu2FLqinBK7WgJFpTICvYe4b2ssKhfe7wp/SfCPGgWZat5boOc\nuPIAoye2UTgcD1pKgEUtEuE7kLsA+ZE/pfZsHDhAhgV9Ij8ni2iNCOPney4ITKWudUewW4DTcXPL\nPszn8zCMjNf7iDCex1wIXs6y9240QpSQCN8RfoxPqSu87AP2Jsfc7/cd8fUsrtpqat7rL5dLq5Qm\nRznYd8i5IjCqILISvX4fuJavV0zd1ufzebXbIQqri/reMtomRA6J8A3p82iNoWjmy42m/pnNZuH7\ncWGcvpawZ9VF74/+VoyZ9dZrHtc5saOGxWIRzmLB6yjCpWXf4xDiVkiEB4Z9xiZOFpmQUrcWsTdv\nnZETbOZyubRcCPYYb+Fw2+02bTabJjY5V/Sc42q9R/hoWx88dwRnsrHf3MtmuyYyQ4h7IBF+ALxH\nXRRifPyfz+ed2hL2d+PxuFfBGBNhi0PGTDxODjERzs2Xhus5Pypv64PdgLzMNp67zctiwybEIyAR\nHhCzgK2Py5T+CDEPgi0Wi1ZRdHNnTCaTtFgsqj//crk0ac9WxQ1rU/CSkyI4OQJbNIjlrfehFKIW\nRZDIEhaPikR4YFgEeH08HjfhbBhXjAN8aCl7GXM5aiqzWT+XyMDLnPB+RATZ4vaSNTwR9twhEmDx\nCEiEHwz0EWMznzAXXkd/scUW14Jpy9iibV6qLzce6MqJ7zVCyIkhkSXuRZFElrDEWAyJRHhgPEuY\nC6mjpcsuCIzT3e/3zUSgNZgI17TT6dQZ6PIGv2ybF96VW6+FbwJeER1rucFAia94FCTCA+L5hKP9\nk0l73joUZpsqyQoB9cHigqNylrgvyh7zfK9RXG0uxraG6CYQhZ3l3CLXfL4Qt0Yi/ABgiFq0zYri\noPh6oskhazkwFpkLu3v9UuwvRx7UJDP0FUF7AijdBNAa91wPsoTFozC61aSLH+QhDuJvkpva3dvH\ns15EiR19RNjeH98jtywNtkUWMPe99VrYpZCLQ46s3pxlLsSNKf5jSYQHonTePWHGltt2zXF478v9\nlOpnysiJ2kcFr8bfzJ9TuglIhMWdkAgLIcSAFEVYaUNCCDEgEmEhhBgQibAQQgyIRFgIIQZEIiyE\nEAMiERZCiAGRCAshxIBIhIUQYkAkwkIIMSASYSGEGBCJsBBCDIhEWAghBkQiLIQQAyIRFkKIAZEI\nCyHEgEiEhRBiQCTCQggxIBJhIYQYEImwEEIMiERYCCEGRCIshBADIhEWQogBkQgLIcSASISFEGJA\nJMJCCDEgEmEhhBgQibAQQgzIdOgD+P+Mhj4AIYQYAlnCQggxIBJhIYQYEImwEEIMiERYCCEGRCIs\nhBADIhEWQogBkQgLIcSASISFEGJAJMJCCDEgEmEhhBgQibAQQgyIRFgIIQZEIiyEEAMiERZCiAGR\nCAshxIBIhIUQYkAkwkIIMSASYSGEGBCJsBBCDIhEWAghBkQiLIQQAyIRFkKIAZEICyHEgEiEhRBi\nQCTCQggxIBJhIYQYEImwEEIMiERYCCEGRCIshBAD8v8AFLWioi3Qk3kAAAAASUVORK5CYII=\n", - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], + "outputs": [], "source": [ "# Plot a random image\n", "sample_number = 5001\n", @@ -223,7 +188,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": { "collapsed": false }, @@ -232,10 +197,10 @@ "# Save the data files into a format compatible with CNTK text reader\n", "def savetxt(filename, ndarray):\n", " dir = os.path.dirname(filename)\n", - " \n", + "\n", " if not os.path.exists(dir):\n", " os.makedirs(dir)\n", - " \n", + "\n", " if not os.path.isfile(filename):\n", " print(\"Saving\", filename )\n", " with open(filename, 'w') as f:\n", @@ -251,30 +216,22 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": { "collapsed": false }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Writing train text file...\n", - "Saving data/MNIST/Train-28x28_cntk_text.txt\n", - "Writing test text file...\n", - "Saving data/MNIST/Test-28x28_cntk_text.txt\n", - "Done\n" - ] - } - ], + "outputs": [], "source": [ - "# Save the train and test files\n", + "# Save the train and test files (prefer our default path for the data)\n", + "data_dir = os.path.join(\"..\", \"Examples\", \"Image\", \"DataSets\", \"MNIST\")\n", + "if not os.path.exists(data_dir):\n", + " data_dir = os.path.join(\"data\", \"MNIST\")\n", + "\n", "print ('Writing train text file...')\n", - "savetxt(r'data/MNIST/Train-28x28_cntk_text.txt', train)\n", + "savetxt(os.path.join(data_dir, \"Train-28x28_cntk_text.txt\"), train)\n", "\n", "print ('Writing test text file...')\n", - "savetxt(r'data/MNIST/Test-28x28_cntk_text.txt', test)\n", + "savetxt(os.path.join(data_dir, \"Test-28x28_cntk_text.txt\"), test)\n", "\n", "print('Done')" ] diff --git a/Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb b/Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb index c2e4965d76ec..97f2cfe51a20 100644 --- a/Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb +++ b/Tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb @@ -58,6 +58,13 @@ "from cntk.initializer import glorot_uniform\n", "from cntk.learner import sgd\n", "from cntk.ops import *\n", + "# Select the right target device when this notebook is being tested:\n", + "if 'TEST_DEVICE' in os.environ:\n", + " import cntk\n", + " if os.environ['TEST_DEVICE'] == 'cpu':\n", + " cntk.device.set_default_device(cntk.device.cpu())\n", + " else:\n", + " cntk.device.set_default_device(cntk.device.gpu(0))\n", "\n", "%matplotlib inline" ] @@ -104,18 +111,25 @@ }, "outputs": [], "source": [ - "# Ensure the training data is generated and available for this tutorial\n", - "train_file = \"data/MNIST/Train-28x28_cntk_text.txt\"\n", - "\n", - "if os.path.isfile(train_file):\n", - " path = train_file\n", - "else:\n", + "# Ensure the training and test data is generated and available for this tutorial.\n", + "# We search in two locations in the toolkit for the cached MNIST data set.\n", + "data_found = False\n", + "for data_dir in [os.path.join(\"..\", \"Examples\", \"Image\", \"DataSets\", \"MNIST\"),\n", + " os.path.join(\"data\", \"MNIST\")]:\n", + " train_file = os.path.join(data_dir, \"Train-28x28_cntk_text.txt\")\n", + " test_file = os.path.join(data_dir, \"Test-28x28_cntk_text.txt\")\n", + " if os.path.isfile(train_file) and os.path.isfile(test_file):\n", + " data_found = True\n", + " break\n", + "if not data_found:\n", " raise ValueError(\"Please generate the data by completing CNTK 103 Part A\")\n", + "print(\"Data directory is {0}\".format(data_dir))\n", "\n", + "# Set up data reading for the training data\n", "feature_stream_name = 'features'\n", "labels_stream_name = 'labels'\n", "\n", - "mb_source = MinibatchSource(CTFDeserializer(path, StreamDefs(\n", + "mb_source = MinibatchSource(CTFDeserializer(train_file, StreamDefs(\n", " features = StreamDef(field='features', shape=input_dim, is_sparse=False),\n", " labels = StreamDef(field='labels', shape=num_output_classes, is_sparse=False)\n", " )))\n", @@ -123,7 +137,7 @@ "features_si = mb_source[feature_stream_name]\n", "labels_si = mb_source[labels_stream_name]\n", "\n", - "print(\"Training data from file {0} successfully read.\".format(path))" + "print(\"Training data successfully read from file {0}.\".format(train_file))" ] }, { @@ -178,11 +192,6 @@ }, "outputs": [], "source": [ - "# The input variable (representing 1 observation, in our example of age and size) $\\bf{x}$ which \n", - "# in this case has a dimension of 2.\n", - "#\n", - "# The label variable has a dimensionality equal to the number of output classes in our case 2.\n", - "\n", "input = input_variable((input_dim), np.float32)\n", "label = input_variable((num_output_classes), np.float32)" ] @@ -502,18 +511,10 @@ }, "outputs": [], "source": [ - "# Ensure the training data is read and available for this tutorial\n", - "test_file = \"data/MNIST/Test-28x28_cntk_text.txt\"\n", - "\n", - "if os.path.isfile(test_file):\n", - " path = test_file\n", - "else:\n", - " print(\"Please generate the data by completing CNTK 103 Part A\")\n", - "\n", "feature_stream_name = 'features'\n", "labels_stream_name = 'labels'\n", "\n", - "test_mb_source = MinibatchSource(CTFDeserializer(path, StreamDefs(\n", + "test_mb_source = MinibatchSource(CTFDeserializer(test_file, StreamDefs(\n", " features = StreamDef(field='features', shape=input_dim, is_sparse=False),\n", " labels = StreamDef(field='labels', shape=num_output_classes, is_sparse=False)\n", " )))\n", @@ -521,7 +522,7 @@ "features_si = test_mb_source[feature_stream_name]\n", "labels_si = test_mb_source[labels_stream_name]\n", "\n", - "print(\"Test data from file {0} successfully read\".format(path))\n" + "print(\"Test data successfully read from file {0}\".format(test_file))\n" ] }, { @@ -670,7 +671,7 @@ "source": [ "#### Code link\n", "\n", - "If you want to try running the tutorial from python command prompt. Please run the [SimpleMNIST.py](https://github.com/Microsoft/CNTK/tree/v2.0.beta7.0/Examples/Image/Classification/MLP/Python) example." + "If you want to try running the tutorial from Python command prompt please run the [SimpleMNIST.py](https://github.com/Microsoft/CNTK/tree/v2.0.beta7.0/Examples/Image/Classification/MLP/Python) example." ] }, { From 1cbc4ed9c6a1e3688829b2cd77ae914e5f813091 Mon Sep 17 00:00:00 2001 From: Nadav Bar Date: Tue, 10 Jan 2017 09:58:09 +0200 Subject: [PATCH 071/120] Fixed an issue with incorrect bounding boxes read when gt annotation contains only one regio --- Examples/Image/Detection/FastRCNN/imdb_data.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Examples/Image/Detection/FastRCNN/imdb_data.py b/Examples/Image/Detection/FastRCNN/imdb_data.py index 45ebe57e174d..1611070c4ef8 100755 --- a/Examples/Image/Detection/FastRCNN/imdb_data.py +++ b/Examples/Image/Detection/FastRCNN/imdb_data.py @@ -152,10 +152,17 @@ def _load_annotation(self, imgIndex): if not os.path.exists(bboxesPaths) or not os.path.exists(labelsPaths): return None bboxes = np.loadtxt(bboxesPaths, np.float32) + + # in case there's only one annotation and numpy read the array as single array, + # we need to make sure the input is treated as a multi dimensional array instead of a list/ 1D array + if len(bboxes.shape) == 1: + bboxes = np.array([bboxes]) + labels = readFile(labelsPaths) #remove boxes marked as 'undecided' or 'exclude' indicesToKeep = find(labels, lambda x: x!='EXCLUDE' and x!='UNDECIDED') + bboxes = [bboxes[i] for i in indicesToKeep] labels = [labels[i] for i in indicesToKeep] From eda6561105ab974cd9b939c264f7b14c6cb75761 Mon Sep 17 00:00:00 2001 From: Nadav Bar Date: Tue, 10 Jan 2017 10:00:00 +0200 Subject: [PATCH 072/120] Remove unnecesarry new line that was mistakenly added --- Examples/Image/Detection/FastRCNN/imdb_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Examples/Image/Detection/FastRCNN/imdb_data.py b/Examples/Image/Detection/FastRCNN/imdb_data.py index 1611070c4ef8..a96563f2f173 100755 --- a/Examples/Image/Detection/FastRCNN/imdb_data.py +++ b/Examples/Image/Detection/FastRCNN/imdb_data.py @@ -162,7 +162,6 @@ def _load_annotation(self, imgIndex): #remove boxes marked as 'undecided' or 'exclude' indicesToKeep = find(labels, lambda x: x!='EXCLUDE' and x!='UNDECIDED') - bboxes = [bboxes[i] for i in indicesToKeep] labels = [labels[i] for i in indicesToKeep] From 911b2ca82e7be1b36ce53c39e8287314c29648fc Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Mon, 2 Jan 2017 12:52:20 +0100 Subject: [PATCH 073/120] Windows: switch to Visual Studio 2015 --- .gitignore | 4 ++ CNTK.Cpp.props | 58 +++++++++---------- CNTK.sln | 4 +- Dependencies/CNTKCustomMKL/build-windows.cmd | 2 +- Dependencies/CNTKCustomMKL/version.txt | 2 +- .../CNTKAzureTutorial01.csproj | 4 +- .../CPPEvalClient/CPPEvalClient.vcxproj | 4 +- .../CPPEvalExtendedClient.vcxproj | 4 +- .../CPPEvalV2Client/CPPEvalV2Client.vcxproj | 4 +- .../CSEvalClient/CSEvalClient.csproj | 2 +- Source/1BitSGD | 2 +- Source/ActionsLib/ActionsLib.vcxproj | 8 ++- Source/CNTK/CNTK.vcxproj | 8 +-- .../CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj | 4 +- Source/CNTKv2LibraryDll/Utils.cpp | 2 +- Source/Common/Common.vcxproj | 4 +- Source/Common/DataReader.cpp | 1 + Source/Common/Include/Platform.h | 6 +- .../ComputationNetworkLib.vcxproj | 2 +- Source/EvalDll/EvalDll.vcxproj | 6 +- .../EvalWrapper/EvalWrapper.vcxproj | 6 +- Source/Math/BlockMultiplier.h | 7 --- Source/Math/CPURNGHandle.cpp | 2 +- Source/Math/CntkBatchNormalization.cuh | 9 +-- Source/Math/GPUMatrixCUDAKernels.cuh | 9 +-- Source/Math/Math.vcxproj | 6 +- Source/Math/MathCUDA.vcxproj | 2 +- Source/Multiverso | 2 +- .../PerformanceProfilerDll.vcxproj | 6 +- .../Readers/BinaryReader/BinaryReader.vcxproj | 6 +- .../CNTKBinaryReader/BinaryConfigHelper.cpp | 6 +- .../CNTKBinaryReader/CNTKBinaryReader.vcxproj | 4 +- .../CNTKTextFormatReader.vcxproj | 4 +- .../CompositeDataReader.vcxproj | 4 +- Source/Readers/DSSMReader/DSSMReader.vcxproj | 6 +- .../HTKDeserializers/HTKDeserializers.vcxproj | 4 +- .../Readers/HTKMLFReader/HTKMLFReader.vcxproj | 6 +- .../Readers/ImageReader/ImageReader.vcxproj | 6 +- .../LMSequenceReader/LMSequenceReader.vcxproj | 6 +- .../LUSequenceReader/LUSequenceReader.vcxproj | 6 +- .../LibSVMBinaryReader.vcxproj | 6 +- Source/Readers/ReaderLib/ReaderLib.vcxproj | 2 +- .../SparsePCReader/SparsePCReader.vcxproj | 6 +- .../UCIFastReader/UCIFastReader.vcxproj | 6 +- Source/SGDLib/SGD.cpp | 2 + Source/SGDLib/SGDLib.vcxproj | 2 +- .../SequenceTrainingLib.vcxproj | 2 +- .../CPPEvalClientTest.vcxproj | 6 +- .../CPPEvalExtendedClientTest.vcxproj | 8 +-- .../CSEvalClientTest/App.config | 6 +- .../CSEvalClientTest/CSEvalClientTest.csproj | 4 +- .../Speech/LSTM/Truncated/testcases.yml | 3 +- .../UnitTests/ManagedEvalTests/run-test | 2 +- .../BrainScriptTests/BrainScriptTests.vcxproj | 4 +- .../UnitTests/CommandEval/CommandEval.vcxproj | 6 +- Tests/UnitTests/EvalTests/EvalTests.vcxproj | 6 +- .../ManagedEvalTests/ManagedEvalTests.csproj | 4 +- .../MathPerformanceTests.vcxproj | 6 +- Tests/UnitTests/MathTests/MathTests.vcxproj | 6 +- .../MatrixSparseDenseInteractionsTests.cpp | 2 - .../NetworkTests/NetworkTests.vcxproj | 6 +- .../UnitTests/ReaderTests/ReaderTests.vcxproj | 4 +- .../V2LibraryDistributionTests.vcxproj | 6 +- .../UserDefinedFunctionTests.cpp | 6 +- .../V2LibraryTests/V2LibraryTests.vcxproj | 6 +- Tools/build-and-test | 4 +- Tools/check-git-head.sh | 2 +- Tools/make_binary_drop_windows.ps1 | 4 ++ Tools/msvc_collect_coverage.py | 4 +- .../CSEvalV2Example/CSEvalV2Example.csproj | 4 +- .../CSEvalV2Library/CSEvalV2Library.csproj | 4 +- bindings/csharp/Swig/CSharpBindings.vcxproj | 6 +- bindings/python/PythonBindings.vcxproj | 4 +- bindings/python/build.bat | 2 +- bindings/python/cntk/tests/persist_test.py | 2 +- bindings/python/setup.py | 7 ++- bindings/python/vsbuild.bat | 6 +- configure | 2 +- 78 files changed, 206 insertions(+), 200 deletions(-) diff --git a/.gitignore b/.gitignore index defcd00b931e..148cbba4605a 100644 --- a/.gitignore +++ b/.gitignore @@ -194,6 +194,10 @@ Tests/EndToEndTests/UnitTests/MathTests/M.txt Tests/EndToEndTests/UnitTests/MathTests/MCPU.txt Tests/EndToEndTests/UnitTests/MathTests/MGPU.txt Tests/EndToEndTests/UnitTests/MathTests/MS.txt +Tests/UnitTests/MathTests/M.txt +Tests/UnitTests/MathTests/MCPU.txt +Tests/UnitTests/MathTests/MGPU.txt +Tests/UnitTests/MathTests/MS.txt Tests/Install/linux/BinaryDrops.tar.gz diff --git a/CNTK.Cpp.props b/CNTK.Cpp.props index fc0a2a7b6d9f..9d01b2da73db 100644 --- a/CNTK.Cpp.props +++ b/CNTK.Cpp.props @@ -23,13 +23,17 @@ false true - "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\include" - "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" - 8.0 7.5 - 7.0 + + + "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\include" + + + + "c:\Program Files\NVIDIA Corporation\GDK\gdk_win7_amd64_release\nvml\lib" + false true @@ -55,27 +59,27 @@ - MKL - 2 - $(CNTK_MKL_PATH)\$(CNTKCustomMKLVersion) - $(CNTKCustomMKLPath)\include - USE_MKL + MKL + 3 + $(CNTK_MKL_PATH)\$(CNTKCustomMKLVersion) + $(CNTKCustomMKLPath)\include + USE_MKL - CNTK custom MKL Parallel (Version: $(CNTKCustomMKLVersion)) - $(CNTKCustomMKLPath)\x64\parallel - mkl_cntk_p.lib - mkl_cntk_p.dll - $(MathLibraryPath)\*.dll - $(OutDir)mkl_cntk_p.dll;$(OutDir)libiomp5md.dll; + CNTK custom MKL Parallel (Version: $(CNTKCustomMKLVersion)) + $(CNTKCustomMKLPath)\x64\parallel + mkl_cntk_p.lib + mkl_cntk_p.dll + $(MathLibraryPath)\*.dll + $(OutDir)mkl_cntk_p.dll;$(OutDir)libiomp5md.dll; - CNTK custom MKL Sequential (Version: $(CNTKCustomMKLVersion)) - $(CNTKCustomMKLPath)\x64\sequential - mkl_cntk_s.lib - mkl_cntk_s.dll - $(MathLibraryPath)\*.dll - $(OutDir)mkl_cntk_s.dll; + CNTK custom MKL Sequential (Version: $(CNTKCustomMKLVersion)) + $(CNTKCustomMKLPath)\x64\sequential + mkl_cntk_s.lib + mkl_cntk_s.dll + $(MathLibraryPath)\*.dll + $(OutDir)mkl_cntk_s.dll; @@ -100,8 +104,8 @@ opencv_world$(OpenCvVersion) opencv_world$(OpenCvVersion)d $(OpenCvWorld).lib - $(OpenCvPath)\x64\vc12\lib - $(OpenCvPath)\x64\vc12\bin + $(OpenCvPath)\x64\vc14\lib + $(OpenCvPath)\x64\vc14\bin @@ -123,12 +127,6 @@ cublas64_75.dll;cusparse64_75.dll;curand64_75.dll;$(CudaRuntimeDll) - - $(CUDA_PATH_V7_0) - cudart64_70.dll - cublas64_70.dll;cusparse64_70.dll;curand64_70.dll;$(CudaRuntimeDll) - - cudart.lib;cublas.lib;cusparse.lib;curand.lib $(CudaPath)\include @@ -139,7 +137,7 @@ $(DebugBuild) - v120 + v140 Unicode $(ReleaseBuild) $(DebugBuild) diff --git a/CNTK.sln b/CNTK.sln index 7f432ed7b2fd..85997249be13 100644 --- a/CNTK.sln +++ b/CNTK.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.40629.0 +# Visual Studio 14 +VisualStudioVersion = 14.0.25420.1 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CNTK", "Source\CNTK\CNTK.vcxproj", "{E6F26F9A-FF64-4F0A-B749-CD309EE357EE}" ProjectSection(ProjectDependencies) = postProject diff --git a/Dependencies/CNTKCustomMKL/build-windows.cmd b/Dependencies/CNTKCustomMKL/build-windows.cmd index fdb36dd3b4fd..38bd7a597644 100644 --- a/Dependencies/CNTKCustomMKL/build-windows.cmd +++ b/Dependencies/CNTKCustomMKL/build-windows.cmd @@ -12,7 +12,7 @@ echo. echo Requirements: echo - Intel MKL SDK installed on the machine echo - MKLROOT environment variable is set to the MKL directory inside the Intel MKL SDK -echo - Visual Studio 2013 installed and included in the path +echo - Visual Studio 2015 installed and included in the path echo. setlocal enableextensions enabledelayedexpansion diff --git a/Dependencies/CNTKCustomMKL/version.txt b/Dependencies/CNTKCustomMKL/version.txt index 0cfbf08886fc..00750edc07d6 100644 --- a/Dependencies/CNTKCustomMKL/version.txt +++ b/Dependencies/CNTKCustomMKL/version.txt @@ -1 +1 @@ -2 +3 diff --git a/Examples/Evaluation/CNTKAzureTutorial01/CNTKAzureTutorial01/CNTKAzureTutorial01.csproj b/Examples/Evaluation/CNTKAzureTutorial01/CNTKAzureTutorial01/CNTKAzureTutorial01.csproj index 61841bcd3092..857125a4937d 100644 --- a/Examples/Evaluation/CNTKAzureTutorial01/CNTKAzureTutorial01/CNTKAzureTutorial01.csproj +++ b/Examples/Evaluation/CNTKAzureTutorial01/CNTKAzureTutorial01/CNTKAzureTutorial01.csproj @@ -1,5 +1,5 @@  - + Debug @@ -163,4 +163,4 @@ --> - \ No newline at end of file + diff --git a/Examples/Evaluation/CPPEvalClient/CPPEvalClient.vcxproj b/Examples/Evaluation/CPPEvalClient/CPPEvalClient.vcxproj index 0914636c2944..37213ab06ad0 100644 --- a/Examples/Evaluation/CPPEvalClient/CPPEvalClient.vcxproj +++ b/Examples/Evaluation/CPPEvalClient/CPPEvalClient.vcxproj @@ -1,5 +1,5 @@  - + Release @@ -16,7 +16,7 @@ Application false - v120 + v140 true Unicode diff --git a/Examples/Evaluation/CPPEvalExtendedClient/CPPEvalExtendedClient.vcxproj b/Examples/Evaluation/CPPEvalExtendedClient/CPPEvalExtendedClient.vcxproj index e2d9ec1bfb0b..900803f633e8 100644 --- a/Examples/Evaluation/CPPEvalExtendedClient/CPPEvalExtendedClient.vcxproj +++ b/Examples/Evaluation/CPPEvalExtendedClient/CPPEvalExtendedClient.vcxproj @@ -1,5 +1,5 @@  - + Release @@ -16,7 +16,7 @@ Application false - v120 + v140 true Unicode diff --git a/Examples/Evaluation/CPPEvalV2Client/CPPEvalV2Client.vcxproj b/Examples/Evaluation/CPPEvalV2Client/CPPEvalV2Client.vcxproj index a76f4a9bd809..89346621eb2c 100644 --- a/Examples/Evaluation/CPPEvalV2Client/CPPEvalV2Client.vcxproj +++ b/Examples/Evaluation/CPPEvalV2Client/CPPEvalV2Client.vcxproj @@ -1,5 +1,5 @@  - + Release @@ -20,7 +20,7 @@ Application false - v120 + v140 true Unicode diff --git a/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj b/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj index e37a45565777..12bfc522a527 100644 --- a/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj +++ b/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj @@ -1,5 +1,5 @@  - + Debug diff --git a/Source/1BitSGD b/Source/1BitSGD index 2184b8af1f67..7bde79e23210 160000 --- a/Source/1BitSGD +++ b/Source/1BitSGD @@ -1 +1 @@ -Subproject commit 2184b8af1f670b330b580b0b5d9d3e9bcb71b08a +Subproject commit 7bde79e23210f87289af940c6b4e615a335f830f diff --git a/Source/ActionsLib/ActionsLib.vcxproj b/Source/ActionsLib/ActionsLib.vcxproj index 1f494721a3b0..7a37620bbf4a 100644 --- a/Source/ActionsLib/ActionsLib.vcxproj +++ b/Source/ActionsLib/ActionsLib.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -59,10 +59,14 @@ Math.dll + + + /bigobj %(AdditionalOptions) + + _SCL_SECURE_NO_WARNINGS;%(PreprocessorDefinitions) - /bigobj %(AdditionalOptions) diff --git a/Source/CNTK/CNTK.vcxproj b/Source/CNTK/CNTK.vcxproj index 430085fb0642..1c741729fcf9 100644 --- a/Source/CNTK/CNTK.vcxproj +++ b/Source/CNTK/CNTK.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ Application true - v120 + v140 Unicode Application false - v120 + v140 true Unicode @@ -229,4 +229,4 @@ - + \ No newline at end of file diff --git a/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj b/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj index 5999bfbfb7fe..dba317fa704b 100644 --- a/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj +++ b/Source/CNTKv2LibraryDll/CNTKv2LibraryDll.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,7 +33,7 @@ DynamicLibrary - v120 + v140 Unicode diff --git a/Source/CNTKv2LibraryDll/Utils.cpp b/Source/CNTKv2LibraryDll/Utils.cpp index 097bd6197933..004c3cd4854c 100644 --- a/Source/CNTKv2LibraryDll/Utils.cpp +++ b/Source/CNTKv2LibraryDll/Utils.cpp @@ -506,7 +506,7 @@ namespace CNTK if ((mask != nullptr) && ((varShape.Rank() + mask->Shape().Rank()) != valueShape.Rank())) InvalidArgument("Invalid Value object; the sum of the rank of the mask and data does not equal the Variable's rank + number of dynamic axes"); - auto getNumTimeStepsAndSequencesFunc = [numDynamicAxes](const NDShape& maskShape, size_t numDynamicAxes) { + auto getNumTimeStepsAndSequencesFunc = [](const NDShape& maskShape, size_t numDynamicAxes) { size_t maxNumTimeSteps = 1; size_t numSequences = 1; if (maskShape.Rank() > 1) diff --git a/Source/Common/Common.vcxproj b/Source/Common/Common.vcxproj index 3ac3d26526c4..613bae397cd7 100644 --- a/Source/Common/Common.vcxproj +++ b/Source/Common/Common.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -77,4 +77,4 @@ - + \ No newline at end of file diff --git a/Source/Common/DataReader.cpp b/Source/Common/DataReader.cpp index 9200a2f4586a..a4ec634f92d4 100644 --- a/Source/Common/DataReader.cpp +++ b/Source/Common/DataReader.cpp @@ -79,6 +79,7 @@ void DataReader::Destroy() // DataReader Constructor // options - [in] string of options (i.e. "-windowsize:11 -addenergy") data reader specific +#pragma optimize("", off) // TODO work around potential VS2015 code optimization bug, replacing virtual- by non-virtual call in Init() below template DataReader::DataReader(const ConfigRecordType& config) { diff --git a/Source/Common/Include/Platform.h b/Source/Common/Include/Platform.h index 932a6b45214a..cd5dde58d6c6 100644 --- a/Source/Common/Include/Platform.h +++ b/Source/Common/Include/Platform.h @@ -11,8 +11,7 @@ #define __UNIX__ #endif -#ifdef _MSC_VER -// TODO: thread_local is supported in VS2015. Remove this macro when we uprade to VS2015 +#if defined(_MSC_VER) && (_MSC_VER <= 1800 /*VS2013*/) #define THREAD_LOCAL __declspec(thread) #else #define THREAD_LOCAL thread_local @@ -38,9 +37,10 @@ #define __func__ __FUNCTION__ #endif -#if defined(_MSC_VER) +#if defined(_MSC_VER) && (_MSC_VER <= 1800 /*VS2013*/) #define snprintf _snprintf #endif + // =========================================================================== // emulation of some MSVC proprietary CRT // =========================================================================== diff --git a/Source/ComputationNetworkLib/ComputationNetworkLib.vcxproj b/Source/ComputationNetworkLib/ComputationNetworkLib.vcxproj index 39e03224c707..d5512c4fd288 100644 --- a/Source/ComputationNetworkLib/ComputationNetworkLib.vcxproj +++ b/Source/ComputationNetworkLib/ComputationNetworkLib.vcxproj @@ -1,5 +1,5 @@  - + Debug diff --git a/Source/EvalDll/EvalDll.vcxproj b/Source/EvalDll/EvalDll.vcxproj index 373599523aa4..0f1b80b49e8c 100644 --- a/Source/EvalDll/EvalDll.vcxproj +++ b/Source/EvalDll/EvalDll.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj b/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj index d6d8f1f79b31..0442f1019535 100644 --- a/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj +++ b/Source/Extensibility/EvalWrapper/EvalWrapper.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,14 +33,14 @@ DynamicLibrary true - v120 + v140 true Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Math/BlockMultiplier.h b/Source/Math/BlockMultiplier.h index e867cc256a99..75f0c01f5c22 100644 --- a/Source/Math/BlockMultiplier.h +++ b/Source/Math/BlockMultiplier.h @@ -583,13 +583,6 @@ template class BlockMultiplier int m_oldNumThreads; }; -// Instantiate block multipliers -template<> const int BlockMultiplier::MAXRANGE; -#ifdef SUPPORT_AVX2 -template<> const int BlockMultiplier::MAXRANGE; -#endif - - template typename BlockMultiplier::ScalarAT* BlockMultiplier::CreateMatrixA(int m, int n, ScalarAT initVal) { return CreateAlignedMatrix(m, n, initVal); diff --git a/Source/Math/CPURNGHandle.cpp b/Source/Math/CPURNGHandle.cpp index 1c7ac15b110b..4aaacf416af8 100644 --- a/Source/Math/CPURNGHandle.cpp +++ b/Source/Math/CPURNGHandle.cpp @@ -13,7 +13,7 @@ namespace Microsoft { namespace MSR { namespace CNTK { CPURNGHandle::CPURNGHandle(int deviceId, uint64_t seed, uint64_t offset) : RNGHandle(deviceId) { - m_generator.reset(new std::mt19937_64((unsigned long)seed)); + m_generator.reset(new std::mt19937_64(seed)); m_generator->discard(offset); } diff --git a/Source/Math/CntkBatchNormalization.cuh b/Source/Math/CntkBatchNormalization.cuh index 34c2b8d9f90b..2a0e16dcf2f5 100644 --- a/Source/Math/CntkBatchNormalization.cuh +++ b/Source/Math/CntkBatchNormalization.cuh @@ -7,10 +7,11 @@ #ifdef _MSC_VER #pragma warning(push) -#pragma warning(disable : 4100) -#pragma warning(disable : 4127) -#pragma warning(disable : 4201) -#pragma warning(disable : 4515) +#pragma warning(disable : 4100) // 'identifier': unreferenced formal parameter +#pragma warning(disable : 4127) // conditional expression is constant +#pragma warning(disable : 4201) // nonstandard extension used: nameless struct/union +#pragma warning(disable : 4458) // declaration of 'identifier' hides class member +#pragma warning(disable : 4515) // 'namespace': namespace uses itself #endif #include #ifdef _MSC_VER diff --git a/Source/Math/GPUMatrixCUDAKernels.cuh b/Source/Math/GPUMatrixCUDAKernels.cuh index 002c781139d3..05f00ed4678c 100644 --- a/Source/Math/GPUMatrixCUDAKernels.cuh +++ b/Source/Math/GPUMatrixCUDAKernels.cuh @@ -23,10 +23,11 @@ // REVIEW alexeyk: disable warnings properly for GCC/clang #ifdef _MSC_VER #pragma warning(push) -#pragma warning(disable : 4100) -#pragma warning(disable : 4127) -#pragma warning(disable : 4201) -#pragma warning(disable : 4515) +#pragma warning(disable : 4100) // 'identifier': unreferenced formal parameter +#pragma warning(disable : 4127) // conditional expression is constant +#pragma warning(disable : 4201) // nonstandard extension used: nameless struct/union +#pragma warning(disable : 4458) // declaration of 'identifier' hides class member +#pragma warning(disable : 4515) // 'namespace': namespace uses itself #endif #include #ifdef _MSC_VER diff --git a/Source/Math/Math.vcxproj b/Source/Math/Math.vcxproj index b120a1aafead..b241b5d50335 100644 --- a/Source/Math/Math.vcxproj +++ b/Source/Math/Math.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 Unicode diff --git a/Source/Math/MathCUDA.vcxproj b/Source/Math/MathCUDA.vcxproj index 6bfe9c5f61e3..bbb835518698 100644 --- a/Source/Math/MathCUDA.vcxproj +++ b/Source/Math/MathCUDA.vcxproj @@ -1,5 +1,5 @@  - + Debug diff --git a/Source/Multiverso b/Source/Multiverso index a1170ce7e7fd..143187575d1c 160000 --- a/Source/Multiverso +++ b/Source/Multiverso @@ -1 +1 @@ -Subproject commit a1170ce7e7fd5dbf44b7552fe1570843877fc4b8 +Subproject commit 143187575d1cfa410100037b8aea2e767e0af637 diff --git a/Source/PerformanceProfilerDll/PerformanceProfilerDll.vcxproj b/Source/PerformanceProfilerDll/PerformanceProfilerDll.vcxproj index 159c70763e0d..123bf66a0e13 100644 --- a/Source/PerformanceProfilerDll/PerformanceProfilerDll.vcxproj +++ b/Source/PerformanceProfilerDll/PerformanceProfilerDll.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Readers/BinaryReader/BinaryReader.vcxproj b/Source/Readers/BinaryReader/BinaryReader.vcxproj index d5dfe337a6bb..0ab0ce13b0bf 100644 --- a/Source/Readers/BinaryReader/BinaryReader.vcxproj +++ b/Source/Readers/BinaryReader/BinaryReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,13 +32,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Readers/CNTKBinaryReader/BinaryConfigHelper.cpp b/Source/Readers/CNTKBinaryReader/BinaryConfigHelper.cpp index 8e5b35ac7ebd..d1f31653b52d 100644 --- a/Source/Readers/CNTKBinaryReader/BinaryConfigHelper.cpp +++ b/Source/Readers/CNTKBinaryReader/BinaryConfigHelper.cpp @@ -31,14 +31,14 @@ namespace Microsoft { namespace MSR { namespace CNTK { for (const pair& section : input) { - ConfigParameters input = section.second; + ConfigParameters sectionConfig = section.second; wstring name = msra::strfun::utf16(section.first); // If there is an option for "alias", we will rename the stream with the "alias" // name to the target name. - if (input.ExistsCurrent(L"alias")) + if (sectionConfig.ExistsCurrent(L"alias")) { - wstring alias = msra::strfun::utf16(input(L"alias")); + wstring alias = msra::strfun::utf16(sectionConfig(L"alias")); m_streams[alias] = name; } else diff --git a/Source/Readers/CNTKBinaryReader/CNTKBinaryReader.vcxproj b/Source/Readers/CNTKBinaryReader/CNTKBinaryReader.vcxproj index 27db76f8fef8..9874d1b47652 100644 --- a/Source/Readers/CNTKBinaryReader/CNTKBinaryReader.vcxproj +++ b/Source/Readers/CNTKBinaryReader/CNTKBinaryReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,7 +32,7 @@ DynamicLibrary - v120 + v140 Unicode diff --git a/Source/Readers/CNTKTextFormatReader/CNTKTextFormatReader.vcxproj b/Source/Readers/CNTKTextFormatReader/CNTKTextFormatReader.vcxproj index 2eed98a75634..4855bba3881c 100644 --- a/Source/Readers/CNTKTextFormatReader/CNTKTextFormatReader.vcxproj +++ b/Source/Readers/CNTKTextFormatReader/CNTKTextFormatReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,7 +32,7 @@ DynamicLibrary - v120 + v140 Unicode diff --git a/Source/Readers/CompositeDataReader/CompositeDataReader.vcxproj b/Source/Readers/CompositeDataReader/CompositeDataReader.vcxproj index d06ca048e660..c01e04c2f758 100644 --- a/Source/Readers/CompositeDataReader/CompositeDataReader.vcxproj +++ b/Source/Readers/CompositeDataReader/CompositeDataReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -31,7 +31,7 @@ DynamicLibrary - v120 + v140 Unicode $(DebugBuild) $(ReleaseBuild) diff --git a/Source/Readers/DSSMReader/DSSMReader.vcxproj b/Source/Readers/DSSMReader/DSSMReader.vcxproj index d06080f58420..620d5f9ce8ca 100644 --- a/Source/Readers/DSSMReader/DSSMReader.vcxproj +++ b/Source/Readers/DSSMReader/DSSMReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,13 +32,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Readers/HTKDeserializers/HTKDeserializers.vcxproj b/Source/Readers/HTKDeserializers/HTKDeserializers.vcxproj index 739ab43ffcb9..57399ad7338b 100644 --- a/Source/Readers/HTKDeserializers/HTKDeserializers.vcxproj +++ b/Source/Readers/HTKDeserializers/HTKDeserializers.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -31,7 +31,7 @@ DynamicLibrary - v120 + v140 Unicode $(DebugBuild) $(ReleaseBuild) diff --git a/Source/Readers/HTKMLFReader/HTKMLFReader.vcxproj b/Source/Readers/HTKMLFReader/HTKMLFReader.vcxproj index 56453f16ea28..0b4cb3dac625 100644 --- a/Source/Readers/HTKMLFReader/HTKMLFReader.vcxproj +++ b/Source/Readers/HTKMLFReader/HTKMLFReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,13 +32,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Readers/ImageReader/ImageReader.vcxproj b/Source/Readers/ImageReader/ImageReader.vcxproj index be4dd9f78431..cd91604d622e 100644 --- a/Source/Readers/ImageReader/ImageReader.vcxproj +++ b/Source/Readers/ImageReader/ImageReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -31,7 +31,7 @@ DynamicLibrary - v120 + v140 Unicode @@ -71,7 +71,7 @@ if "$(HasOpenCv)" == "true" xcopy /I /D /Y "$(OpenCvBinPath)\$(OpenCvWorld).dll" "$(TargetDir)" if "$(UseZip)" == "true" xcopy /I /D /Y "$(ZLIB_PATH)\bin\zip.dll" "$(TargetDir)" - if "$(UseZip)" == "true" if exist "$(ZLIB_PATH)\bin\zlib1.dll" (xcopy /I /D /Y "$(ZLIB_PATH)\bin\zlib1.dll" "$(TargetDir)") else (copy /Y "$(ZLIB_PATH)\bin\zlib.dll" "$(TargetDir)\zlib1.dll") + if "$(UseZip)" == "true" xcopy /I /D /Y "$(ZLIB_PATH)\bin\zlib.dll" "$(TargetDir)" Copying dependencies diff --git a/Source/Readers/LMSequenceReader/LMSequenceReader.vcxproj b/Source/Readers/LMSequenceReader/LMSequenceReader.vcxproj index b4056c8e7367..090b8150a85e 100644 --- a/Source/Readers/LMSequenceReader/LMSequenceReader.vcxproj +++ b/Source/Readers/LMSequenceReader/LMSequenceReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj b/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj index e39b67b67d86..90c81ff5d4cf 100644 --- a/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj +++ b/Source/Readers/LUSequenceReader/LUSequenceReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj b/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj index afd3f48b8432..598dc9024475 100644 --- a/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj +++ b/Source/Readers/LibSVMBinaryReader/LibSVMBinaryReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,13 +32,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Readers/ReaderLib/ReaderLib.vcxproj b/Source/Readers/ReaderLib/ReaderLib.vcxproj index 0cdfbc1110ab..dd5e066a2bc0 100644 --- a/Source/Readers/ReaderLib/ReaderLib.vcxproj +++ b/Source/Readers/ReaderLib/ReaderLib.vcxproj @@ -1,5 +1,5 @@  - + Debug diff --git a/Source/Readers/SparsePCReader/SparsePCReader.vcxproj b/Source/Readers/SparsePCReader/SparsePCReader.vcxproj index a3200c0b3df9..28464876aa94 100644 --- a/Source/Readers/SparsePCReader/SparsePCReader.vcxproj +++ b/Source/Readers/SparsePCReader/SparsePCReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,13 +32,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/Readers/UCIFastReader/UCIFastReader.vcxproj b/Source/Readers/UCIFastReader/UCIFastReader.vcxproj index fb79bf8b0ba3..d2a77fb829c9 100644 --- a/Source/Readers/UCIFastReader/UCIFastReader.vcxproj +++ b/Source/Readers/UCIFastReader/UCIFastReader.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,13 +32,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/Source/SGDLib/SGD.cpp b/Source/SGDLib/SGD.cpp index 1d6ca807e83c..e17ec185733e 100644 --- a/Source/SGDLib/SGD.cpp +++ b/Source/SGDLib/SGD.cpp @@ -21,8 +21,10 @@ //static inline bool operator==(const std::pair& a, double b) { assert(b==0); return a.first == b; } // ^^ workaround until this line in AggregateGradientsImpl() gets updated: assert(headerCPU->evalErrors[i] == 0); #include "AllReduceDistGradAggregator.h" + #include "BlockMomentumSGD.h" #include "V2BlockMomentumSGD.h" + #include "V2AllReduceDistGradAggregator.h" #endif diff --git a/Source/SGDLib/SGDLib.vcxproj b/Source/SGDLib/SGDLib.vcxproj index 97c915ce448f..c5ba52e63275 100644 --- a/Source/SGDLib/SGDLib.vcxproj +++ b/Source/SGDLib/SGDLib.vcxproj @@ -1,5 +1,5 @@  - + Debug diff --git a/Source/SequenceTrainingLib/SequenceTrainingLib.vcxproj b/Source/SequenceTrainingLib/SequenceTrainingLib.vcxproj index 9725d9eb39a7..99b38e36ffd9 100644 --- a/Source/SequenceTrainingLib/SequenceTrainingLib.vcxproj +++ b/Source/SequenceTrainingLib/SequenceTrainingLib.vcxproj @@ -1,5 +1,5 @@  - + Debug diff --git a/Tests/EndToEndTests/EvalClientTests/CPPEvalClientTest/CPPEvalClientTest.vcxproj b/Tests/EndToEndTests/EvalClientTests/CPPEvalClientTest/CPPEvalClientTest.vcxproj index e7a22b17310f..165754127739 100644 --- a/Tests/EndToEndTests/EvalClientTests/CPPEvalClientTest/CPPEvalClientTest.vcxproj +++ b/Tests/EndToEndTests/EvalClientTests/CPPEvalClientTest/CPPEvalClientTest.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,14 +32,14 @@ Application true - v120 + v140 Unicode No Application false - v120 + v140 true Unicode No diff --git a/Tests/EndToEndTests/EvalClientTests/CPPEvalExtendedClientTest/CPPEvalExtendedClientTest.vcxproj b/Tests/EndToEndTests/EvalClientTests/CPPEvalExtendedClientTest/CPPEvalExtendedClientTest.vcxproj index b9990c0272f4..0792818aa04f 100644 --- a/Tests/EndToEndTests/EvalClientTests/CPPEvalExtendedClientTest/CPPEvalExtendedClientTest.vcxproj +++ b/Tests/EndToEndTests/EvalClientTests/CPPEvalExtendedClientTest/CPPEvalExtendedClientTest.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,14 +32,14 @@ Application true - v120 + v140 Unicode No Application false - v120 + v140 true Unicode No @@ -114,4 +114,4 @@ - + \ No newline at end of file diff --git a/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/App.config b/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/App.config index 8e15646352ec..d1428ad712d7 100644 --- a/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/App.config +++ b/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/App.config @@ -1,6 +1,6 @@ - + - + - \ No newline at end of file + diff --git a/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/CSEvalClientTest.csproj b/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/CSEvalClientTest.csproj index d09ecde929d2..8a1727c6e1fe 100644 --- a/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/CSEvalClientTest.csproj +++ b/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/CSEvalClientTest.csproj @@ -1,5 +1,5 @@  - + Debug @@ -82,4 +82,4 @@ - \ No newline at end of file + diff --git a/Tests/EndToEndTests/Speech/LSTM/Truncated/testcases.yml b/Tests/EndToEndTests/Speech/LSTM/Truncated/testcases.yml index 4e5e1ddb0b06..602b1994589c 100644 --- a/Tests/EndToEndTests/Speech/LSTM/Truncated/testcases.yml +++ b/Tests/EndToEndTests/Speech/LSTM/Truncated/testcases.yml @@ -1,7 +1,8 @@ dataDir: ../../Data tags: - bvt-l (build_sku == 'gpu') and ((flavor=='debug') ^ (device=='cpu')) - - nightly-l (build_sku == 'gpu') + # Note: Windows-Debug-CPU runs for too long (~ 2 hours depending on machine) + - nightly-l (build_sku == 'gpu') and ((flavor=='release') or (os=='linux') or (device=='gpu')) testCases: CNTK Run must be completed: diff --git a/Tests/EndToEndTests/UnitTests/ManagedEvalTests/run-test b/Tests/EndToEndTests/UnitTests/ManagedEvalTests/run-test index 07bc07b25c52..ae97578aaaa3 100755 --- a/Tests/EndToEndTests/UnitTests/ManagedEvalTests/run-test +++ b/Tests/EndToEndTests/UnitTests/ManagedEvalTests/run-test @@ -2,7 +2,7 @@ . $TEST_ROOT_DIR/run-test-common -VSTEST_CONSOLE=$(cygpath -au "$VS120COMNTOOLS\\..\\IDE\\CommonExtensions\\Microsoft\\TestWindow\\vstest.console.exe") +VSTEST_CONSOLE=$(cygpath -au "$VS140COMNTOOLS\\..\\IDE\\CommonExtensions\\Microsoft\\TestWindow\\vstest.console.exe") test -x "$VSTEST_CONSOLE" || exit $? diff --git a/Tests/UnitTests/BrainScriptTests/BrainScriptTests.vcxproj b/Tests/UnitTests/BrainScriptTests/BrainScriptTests.vcxproj index 04f61ae7a69c..cfd5f62f626a 100644 --- a/Tests/UnitTests/BrainScriptTests/BrainScriptTests.vcxproj +++ b/Tests/UnitTests/BrainScriptTests/BrainScriptTests.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,7 +32,7 @@ Application - v120 + v140 Unicode false diff --git a/Tests/UnitTests/CommandEval/CommandEval.vcxproj b/Tests/UnitTests/CommandEval/CommandEval.vcxproj index dfde0f912f82..963a47c527b4 100644 --- a/Tests/UnitTests/CommandEval/CommandEval.vcxproj +++ b/Tests/UnitTests/CommandEval/CommandEval.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ Application true - v120 + v140 Unicode Application false - v120 + v140 true Unicode diff --git a/Tests/UnitTests/EvalTests/EvalTests.vcxproj b/Tests/UnitTests/EvalTests/EvalTests.vcxproj index a482de97d690..ba2eafadf0ea 100644 --- a/Tests/UnitTests/EvalTests/EvalTests.vcxproj +++ b/Tests/UnitTests/EvalTests/EvalTests.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,14 +33,14 @@ Application true - v120 + v140 Unicode false Application false - v120 + v140 true Unicode false diff --git a/Tests/UnitTests/ManagedEvalTests/ManagedEvalTests.csproj b/Tests/UnitTests/ManagedEvalTests/ManagedEvalTests.csproj index 67c95ae647e0..21e21a742d03 100644 --- a/Tests/UnitTests/ManagedEvalTests/ManagedEvalTests.csproj +++ b/Tests/UnitTests/ManagedEvalTests/ManagedEvalTests.csproj @@ -1,5 +1,5 @@  - + Debug AnyCPU @@ -119,4 +119,4 @@ --> - \ No newline at end of file + diff --git a/Tests/UnitTests/MathPerformanceTests/MathPerformanceTests.vcxproj b/Tests/UnitTests/MathPerformanceTests/MathPerformanceTests.vcxproj index c814dc27444a..27e233dc5390 100644 --- a/Tests/UnitTests/MathPerformanceTests/MathPerformanceTests.vcxproj +++ b/Tests/UnitTests/MathPerformanceTests/MathPerformanceTests.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,13 +32,13 @@ Application true - v120 + v140 Unicode Application false - v120 + v140 true Unicode diff --git a/Tests/UnitTests/MathTests/MathTests.vcxproj b/Tests/UnitTests/MathTests/MathTests.vcxproj index e723347295a9..4147028e556a 100644 --- a/Tests/UnitTests/MathTests/MathTests.vcxproj +++ b/Tests/UnitTests/MathTests/MathTests.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,14 +32,14 @@ Application true - v120 + v140 Unicode false Application false - v120 + v140 true Unicode false diff --git a/Tests/UnitTests/MathTests/MatrixSparseDenseInteractionsTests.cpp b/Tests/UnitTests/MathTests/MatrixSparseDenseInteractionsTests.cpp index 6d80283cfddf..6e0017a23c57 100644 --- a/Tests/UnitTests/MathTests/MatrixSparseDenseInteractionsTests.cpp +++ b/Tests/UnitTests/MathTests/MatrixSparseDenseInteractionsTests.cpp @@ -136,8 +136,6 @@ BOOST_FIXTURE_TEST_CASE(MatrixDenseTimesSparse, RandomSeedFixture) BOOST_FIXTURE_TEST_CASE(CPUMatrixDenseTimesSparse, RandomSeedFixture) { - // TODO: test fails with large dimensions - size_t dim1 = 4, dim2 = 2; Matrix mAdense(CPUDEVICE); mAdense.AssignTruncateBottomOf(Matrix::RandomUniform(dim1, dim2, c_deviceIdZero, -3.0f, 0.1f, IncrementCounter()), 0); diff --git a/Tests/UnitTests/NetworkTests/NetworkTests.vcxproj b/Tests/UnitTests/NetworkTests/NetworkTests.vcxproj index 9b3013566d56..4c1179af8b45 100644 --- a/Tests/UnitTests/NetworkTests/NetworkTests.vcxproj +++ b/Tests/UnitTests/NetworkTests/NetworkTests.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -32,7 +32,7 @@ Application - v120 + v140 Unicode false @@ -129,4 +129,4 @@ - + \ No newline at end of file diff --git a/Tests/UnitTests/ReaderTests/ReaderTests.vcxproj b/Tests/UnitTests/ReaderTests/ReaderTests.vcxproj index 81279daba37c..fc2686da7ad8 100644 --- a/Tests/UnitTests/ReaderTests/ReaderTests.vcxproj +++ b/Tests/UnitTests/ReaderTests/ReaderTests.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -35,7 +35,7 @@ Application - v120 + v140 Unicode false diff --git a/Tests/UnitTests/V2LibraryDistributionTests/V2LibraryDistributionTests.vcxproj b/Tests/UnitTests/V2LibraryDistributionTests/V2LibraryDistributionTests.vcxproj index 587dc416bb75..917ca32a7345 100644 --- a/Tests/UnitTests/V2LibraryDistributionTests/V2LibraryDistributionTests.vcxproj +++ b/Tests/UnitTests/V2LibraryDistributionTests/V2LibraryDistributionTests.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ Application true - v120 + v140 Unicode Application false - v120 + v140 true Unicode diff --git a/Tests/UnitTests/V2LibraryTests/UserDefinedFunctionTests.cpp b/Tests/UnitTests/V2LibraryTests/UserDefinedFunctionTests.cpp index 6c736463db67..b607053b47d7 100644 --- a/Tests/UnitTests/V2LibraryTests/UserDefinedFunctionTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/UserDefinedFunctionTests.cpp @@ -339,9 +339,9 @@ void TestDuplicateVariablesInInputs(size_t dim, const DeviceDescriptor& device) // Verify backward prop results if (device.Type() != DeviceKind::CPU) { - NDArrayViewPtr cpuArrayView = MakeSharedObject(DataType::Float, inputShape, DeviceDescriptor::CPUDevice()); - cpuArrayView->CopyFrom(*inputGradientValue->Data()); - const float* cpuArrayViewBuffer = cpuArrayView->DataBuffer(); + NDArrayViewPtr cpuArrayViewBack = MakeSharedObject(DataType::Float, inputShape, DeviceDescriptor::CPUDevice()); + cpuArrayViewBack->CopyFrom(*inputGradientValue->Data()); + const float* cpuArrayViewBuffer = cpuArrayViewBack->DataBuffer(); memcpy(inputGradientData.data(), cpuArrayViewBuffer, inputShape.TotalSize() * sizeof(float)); } diff --git a/Tests/UnitTests/V2LibraryTests/V2LibraryTests.vcxproj b/Tests/UnitTests/V2LibraryTests/V2LibraryTests.vcxproj index b5ef3236ad3d..ad77254c2cb4 100644 --- a/Tests/UnitTests/V2LibraryTests/V2LibraryTests.vcxproj +++ b/Tests/UnitTests/V2LibraryTests/V2LibraryTests.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -33,13 +33,13 @@ Application true - v120 + v140 Unicode Application false - v120 + v140 true Unicode diff --git a/Tools/build-and-test b/Tools/build-and-test index 855abc36dac4..209e70a023a0 100755 --- a/Tools/build-and-test +++ b/Tools/build-and-test @@ -131,8 +131,8 @@ if [[ $OS == "Windows_NT" && $OSTYPE == "cygwin" ]]; then BIN_NAME=CNTK.exe BUILD_OS="windows" - if [[ $VS120COMNTOOLS == "" ]]; then - echo "============ Visual Studio 12.0 environment not properly setup or VS not installed ============" + if [[ $VS140COMNTOOLS == "" ]]; then + echo "============ Visual Studio 14.0 environment not properly setup or VS not installed ============" echo "============ Please find and run the appropriate vcvarsall.bat script ============" exit 1 fi diff --git a/Tools/check-git-head.sh b/Tools/check-git-head.sh index a92ed8711bae..2d17498c7f24 100755 --- a/Tools/check-git-head.sh +++ b/Tools/check-git-head.sh @@ -34,7 +34,7 @@ checkEmptyStdout \ "git ls-tree: path names will illegal characters encountered:" checkEmptyStdout \ - "git grep -l \$'\t' $gitTree -- *.cpp *.h *.cu *.bat *.bs | cut -d: -f2-" \ + "git grep -l \$'\t' $gitTree -- *.cpp *.h *.cu *.bat *.bs CNTK.Cpp.props | cut -d: -f2-" \ "files with hard tabs encountered" checkEmptyStdout \ diff --git a/Tools/make_binary_drop_windows.ps1 b/Tools/make_binary_drop_windows.ps1 index 90bb0fe16a5e..d4d29c811062 100644 --- a/Tools/make_binary_drop_windows.ps1 +++ b/Tools/make_binary_drop_windows.ps1 @@ -84,6 +84,10 @@ If (Test-Path $baseDropPath\cntk\Python\cntk-*-cp27*.whl) { Remove-Item $baseDropPath\cntk\Python\cntk-*-cp27*.whl } +If (Test-Path $baseDropPath\cntk\Python\cntk-*-cp35*.whl) +{ + Remove-Item $baseDropPath\cntk\Python\cntk-*-cp35*.whl +} If (Test-Path $baseDropPath\cntk\CPPEvalClientTest.exe) { Remove-Item $baseDropPath\cntk\CPPEvalClientTest.exe diff --git a/Tools/msvc_collect_coverage.py b/Tools/msvc_collect_coverage.py index 250a0c9fa7d1..5053fa071b97 100755 --- a/Tools/msvc_collect_coverage.py +++ b/Tools/msvc_collect_coverage.py @@ -28,7 +28,7 @@ def collectCoverageMulti(testDir, outputDir, toolDir, config): parser = argparse.ArgumentParser(description="Collects coverage for the executable or directory with test executables") parser.add_argument('--test', help='Path to the executable or directory that has to be analyzed', required=True) parser.add_argument('--outputdir', help='Output directory for coverage results', required=True) - parser.add_argument('--tooldir', help='Tool directory for CodeCoverage tool', required=False, default=r'c:\Program Files (x86)\Microsoft Visual Studio 12.0\Team Tools\Dynamic Code Coverage Tools\amd64') + parser.add_argument('--tooldir', help='Tool directory for CodeCoverage tool', required=False, default=r'c:\Program Files (x86)\Microsoft Visual Studio 14.0\Team Tools\Dynamic Code Coverage Tools\amd64') parser.add_argument('--config', help='Configuration for CodeCoverage tool', required=False, default="") args = parser.parse_args() @@ -41,4 +41,4 @@ def collectCoverageMulti(testDir, outputDir, toolDir, config): elif os.path.isdir(args.test): collectCoverageMulti(args.test, args.outputdir, args.tooldir, args.config) else: - print('Please specify correct executable or test directory where the coverage should be collected.') \ No newline at end of file + print('Please specify correct executable or test directory where the coverage should be collected.') diff --git a/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj b/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj index e5991427b9d1..557d10280342 100644 --- a/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj +++ b/bindings/csharp/CSEvalV2Example/CSEvalV2Example.csproj @@ -1,5 +1,5 @@  - + Debug @@ -87,4 +87,4 @@ --> - \ No newline at end of file + diff --git a/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj b/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj index f7d4159854de..470e5a35d1ad 100644 --- a/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj +++ b/bindings/csharp/CSEvalV2Library/CSEvalV2Library.csproj @@ -1,5 +1,5 @@  - + Debug @@ -118,4 +118,4 @@ --> - \ No newline at end of file + diff --git a/bindings/csharp/Swig/CSharpBindings.vcxproj b/bindings/csharp/Swig/CSharpBindings.vcxproj index 95052f240a03..23da7f9cec95 100644 --- a/bindings/csharp/Swig/CSharpBindings.vcxproj +++ b/bindings/csharp/Swig/CSharpBindings.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -30,13 +30,13 @@ DynamicLibrary true - v120 + v140 Unicode DynamicLibrary false - v120 + v140 true Unicode diff --git a/bindings/python/PythonBindings.vcxproj b/bindings/python/PythonBindings.vcxproj index 6fad1e3c9e9d..6593da1915bf 100644 --- a/bindings/python/PythonBindings.vcxproj +++ b/bindings/python/PythonBindings.vcxproj @@ -1,5 +1,5 @@  - + Debug @@ -30,7 +30,7 @@ Makefile - v120 + v140 $(DebugBuild) diff --git a/bindings/python/build.bat b/bindings/python/build.bat index ba3c704a340f..e166954fac09 100644 --- a/bindings/python/build.bat +++ b/bindings/python/build.bat @@ -2,7 +2,7 @@ setlocal cd "%~dp0" -call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64 +call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall" amd64 set MSSdk=1 set DISTUTILS_USE_SDK=1 diff --git a/bindings/python/cntk/tests/persist_test.py b/bindings/python/cntk/tests/persist_test.py index c249ec541785..b0b2efbf3f8d 100644 --- a/bindings/python/cntk/tests/persist_test.py +++ b/bindings/python/cntk/tests/persist_test.py @@ -46,7 +46,7 @@ def test_load_save_input(tmpdir): loaded_node = load_model(filename) - # Test spefying the input node names by order + # Test specifying the input node names by order loaded_result = loaded_node.eval([input1]) assert np.allclose(loaded_result, expected) diff --git a/bindings/python/setup.py b/bindings/python/setup.py index e1187cfb6bc7..cdcd52df7464 100644 --- a/bindings/python/setup.py +++ b/bindings/python/setup.py @@ -16,9 +16,10 @@ if IS_WINDOWS: if os.system('cl 1>%s 2>%s' % (os.devnull, os.devnull)) != 0: - print("Compiler was not found in path. Please run this from a Visual Studio 2013 x64 Native Tools Command Prompt,\n" - "e.g., by running the following command:\n" - " \"C:\\Program Files (x86)\\Microsoft Visual Studio 12.0\\VC\\vcvarsall\" amd64\n") + print("Compiler was not found in path.\n" + "Make sure you installed the C++ tools during Visual Studio 2015 install and \n" + "run vcvarsall.bat from a DOS command prompt:\n" + " \"C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\vcvarsall\" amd64\n") sys.exit(1) try: diff --git a/bindings/python/vsbuild.bat b/bindings/python/vsbuild.bat index 3fa1450187e1..f01ab8817525 100644 --- a/bindings/python/vsbuild.bat +++ b/bindings/python/vsbuild.bat @@ -31,10 +31,8 @@ if not defined p_CNTK_PY_VERSIONS ( ) REM Validate p_CNTK_PY_VERSIONS contents. -REM TODO no Python 3.5 for now for %%p in (%p_CNTK_PY_VERSIONS%) do ( - if not "%%~p" == "27" if not "%%~p" == "34" echo Build for unsupported Python version '%%~p' requested, stopping&exit /b 1 - set nothingToBuild= + if not "%%~p" == "27" if not "%%~p" == "34" if not "%%~p" == "35" echo Build for unsupported Python version '%%~p' requested, stopping&exit /b 1 ) REM Validate p_CNTK_PY_VERSIONS contents. @@ -49,7 +47,7 @@ if defined nothingToBuild echo Python support not configured to build.&exit /b 0 if "%p_DebugBuild%" == "true" echo Currently no Python build for Debug configurations, exiting.&exit /b 0 -call "C:\Program Files (x86)\Microsoft Visual Studio 12.0\VC\vcvarsall" amd64 +call "C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\vcvarsall" amd64 set CNTK_LIB_PATH=%p_OutDir% set DIST_DIR=%p_OutDir%\Python set PATH=%p_SWIG_PATH%;%PATH% diff --git a/configure b/configure index 9704c24d2421..fc80b0778876 100755 --- a/configure +++ b/configure @@ -22,7 +22,7 @@ nccl_path= nccl_check=include/nccl.h # CNTK Custom MKL Version -cntk_custom_mkl_version=2 +cntk_custom_mkl_version=3 have_mkl=no mkl_path= From 25428483d2957ffca78e74bb7b93e781b75fa09a Mon Sep 17 00:00:00 2001 From: Eldar Akchurin Date: Tue, 10 Jan 2017 10:38:30 +0100 Subject: [PATCH 074/120] Enable better logging in GPU transferer --- Source/Math/GPUDataTransferer.cpp | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/Source/Math/GPUDataTransferer.cpp b/Source/Math/GPUDataTransferer.cpp index 173cef717a6b..db6fdbff8c62 100644 --- a/Source/Math/GPUDataTransferer.cpp +++ b/Source/Math/GPUDataTransferer.cpp @@ -211,23 +211,18 @@ PrefetchGPUDataTransferer::~PrefetchGPUDataTransferer() { PrepareDevice(m_deviceId); } - catch (...) + catch (...) { // the error is already logged return; } - try + auto code = cudaStreamDestroy(m_stream); + if (code != cudaSuccess) { - cudaStreamDestroy(m_stream) || "cudaStreamDestroy failed (PrefetchGPUDataTransferer dtor)"; + std::cerr << "cudaStreamDestroy failed (PrefetchGPUDataTransferer dtor): " + << cudaGetErrorString(code) << " (cuda error " << code << ")"<< std::endl; } - catch (const std::exception& e) - { - // log, but do not re-throw - std::cerr << e.what() << std::endl; - } - catch (...) { } - } }}} From 6b94f75bbf81bb35194fb80321503b393e155bc0 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Thu, 5 Jan 2017 12:07:38 +0100 Subject: [PATCH 075/120] Initial DevInstall version --- Scripts/README.md | 10 +- Scripts/devInstall/Windows/DevInstall.ps1 | 188 +++++ Scripts/devInstall/Windows/helper/Action.ps1 | 782 ++++++++++++++++++ Scripts/devInstall/Windows/helper/Common.ps1 | 42 + Scripts/devInstall/Windows/helper/Display.ps1 | 149 ++++ .../devInstall/Windows/helper/Download.ps1 | 298 +++++++ .../devInstall/Windows/helper/Operations.ps1 | 662 +++++++++++++++ .../Windows/helper/PreRequisites.ps1 | 95 +++ .../Windows/helper/Verification.ps1 | 366 ++++++++ Scripts/devInstall/Windows/readme.md | 118 +++ 10 files changed, 2708 insertions(+), 2 deletions(-) create mode 100644 Scripts/devInstall/Windows/DevInstall.ps1 create mode 100644 Scripts/devInstall/Windows/helper/Action.ps1 create mode 100644 Scripts/devInstall/Windows/helper/Common.ps1 create mode 100644 Scripts/devInstall/Windows/helper/Display.ps1 create mode 100644 Scripts/devInstall/Windows/helper/Download.ps1 create mode 100644 Scripts/devInstall/Windows/helper/Operations.ps1 create mode 100644 Scripts/devInstall/Windows/helper/PreRequisites.ps1 create mode 100644 Scripts/devInstall/Windows/helper/Verification.ps1 create mode 100644 Scripts/devInstall/Windows/readme.md diff --git a/Scripts/README.md b/Scripts/README.md index 10ba16ee1b24..394bca63f0d4 100644 --- a/Scripts/README.md +++ b/Scripts/README.md @@ -1,13 +1,19 @@ -This directory contains different scripts to support CNTK. +This directory contains different scripts to support CNTK. ## CNTK Binary Installers The directory `install` contains scripts which are used in the CNTK binary download to install -CNTK on a users system. They are not intended to run from this location in the repository. +CNTK on a users system. They are NOT intended to run from this location in the repository. * `install/windows` - A script for installing a Windows CNTK *binary* drop, cf. [here](https://github.com/Microsoft/CNTK/wiki/Setup-Windows-Binary-Script). * `install/linux` - A script for installing a Linux CNTK *binary* drop, cf. [here](https://github.com/Microsoft/CNTK/wiki/Setup-Linux-Binary-Script). +## CNTK Development Environment Installer + +The directory `devInstall` contains scripts which are used to create setup an environment to build CNTK from source. They are intended to run from this location in the repository. + +* `devInstall/Windows` - Create a Visual Studio based development environment on Windows + ## CNTK Text format Converters Two Python Scripts for converting Data to CNTK Text format for using as an input for CNTK Text Format diff --git a/Scripts/devInstall/Windows/DevInstall.ps1 b/Scripts/devInstall/Windows/DevInstall.ps1 new file mode 100644 index 000000000000..ffa06f536704 --- /dev/null +++ b/Scripts/devInstall/Windows/DevInstall.ps1 @@ -0,0 +1,188 @@ +<# + .SYNOPSIS + Use this cmdlet to install a CNTK development environment on your machine + + .DESCRIPTION + The script will download and install the files necessary to create a CNTK development environment on your system. + + It will analyse your machine and will determine which components are required. + The required components will be downloaded in [c:\installCacheCntk] + Repeated operation of this script will reuse already downloaded components. + + .PARAMETER Execute + This is an optional parameter. Without setting this switch, no changes to the machine setup/installation will be performed + + .PARAMETER NoGpu + This is an optional parameter. By setting this switch the GPU specific tools (Cuda, CuDnn, Cub) will not be installed. + + .PARAMETER localCache + This optional parameter can be used to specify the directory downloaded components will be stored in + + .PARAMETER ServerLocation + This is an optional parameter. The script can install pre-compiled components, this parameter + specifies the location on a server where this componentents are downloaded from. + This is useful for a team environment to share the components which need to get compiled (Protobuf, Zlib, libzip) + + .PARAMETER CloneDirectory + By default the installer should be executed out of the \Scripts\devInstall\Windows directory. Out of this + location the installer computes the root directory of the CNTK clone. In the case the installer is in a different location, + the root directory of the CNTK clone can be specified using this parameter + +.EXAMPLE + installer.ps1 + + Run the installer and see what operations would be performed +.EXAMPLE + installer.ps1 -Execute + + Run the installer and install the development tools +.EXAMPLE + installer.ps1 -Execute -NoGpu + + Run the installer, but don't install any GPU specific tools +.EXAMPLE + installer.ps1 -Execute -NoGpu -CloneDirectory c:\repos\CNTKAlternate + + Run the installer, but don't install any GPU specific tools +.EXAMPLE + +#> + +[CmdletBinding()] +Param( + [parameter(Mandatory=$false)] [switch] $Execute, + [parameter(Mandatory=$false)] [switch] $NoGpu, + [parameter(Mandatory=$false)] [string] $localCache = "c:\installCacheCntk", + [parameter(Mandatory=$false)] [string] $InstallLocation = "c:\local", + [parameter(Mandatory=$false)] [string] $ServerLocation, + [parameter(Mandatory=$false)] [string] $CloneDirectory) + + + + $Execute = $true + + +$roboCopyCmd = "robocopy.exe" +$localDir = $InstallLocation + + +# Get the current script's directory and Dot-source the a file with common Powershell script function residing in the the current script's directory +$MyDir = Split-Path $MyInvocation.MyCommand.Definition + +if ($CloneDirectory) { + $reponame = Split-Path $CloneDirectory -Leaf + $repositoryRootDir = Split-Path $CloneDirectory + + $solutionfile = join-path $clontDirectory "CNTK.SLN" + + if (-not (Test-Path -Path $solutionFile -PathType Leaf)) { + Write-Warning "[$CloneDirectory] was specified as the location of the CNTK sourcecode directory." + Write-Warning "The specified directory is not a valid clone of the CNTK Github project." + throw "Terminating install operation" + } +} +else { + $CloneDirectory = Split-Path $mydir + $CloneDirectory = Split-Path $CloneDirectory + $CloneDirectory = Split-Path $CloneDirectory + + $reponame = Split-Path $CloneDirectory -Leaf + $repositoryRootDir = Split-Path $CloneDirectory + + $solutionfile = join-path $CloneDirectory "CNTK.SLN" + + if (-not (Test-Path -Path $solutionFile -PathType Leaf)) { + Write-Warning "The install script was started out of the [$mydir] location. Based on this" + Write-Warning "[$CloneDirectory] should be the location of the CNTK sourcecode directory." + Write-Warning "The specified directory is not a valid clone of the CNTK Github project." + throw "Terminating install operation" + } +} + +. "$MyDir\helper\Display" +. "$MyDir\helper\Common" +. "$MyDir\helper\Operations" +. "$MyDir\helper\Verification" +. "$MyDir\helper\Download" +. "$MyDir\helper\Action" +. "$MyDir\helper\PreRequisites" + +Function main +{ + try { if (-not (DisplayStart)) { + Write-Host + Write-Host " ... Quitting ... " + Write-Host + return + } + + if (-not (Test-Path -Path $localCache)) { + new-item -Path $localcache -ItemType Container -ErrorAction Stop | Out-Null + } + + ClearScriptVariables + + $operation = @(); + $operation += OpScanProgram + $operation += OpCheckVS15Update3 + + if (-not $NoGpu) { + $operation += OpCheckCuda8 + + $operation += OpNVidiaCudnn5180 -cache $localCache -targetFolder $localDir + $operation += OpNvidiaCub141 -cache $localCache -targetFolder $localDir + } + + $operation += OpCMake362 -cache $localCache + $operation += OpMSMPI70 -cache $localCache + $operation += OpMSMPI70SDK -cache $localCache + $operation += OpBoost160VS15 -cache $localCache -targetFolder $localDir + $operation += OpCNTKMKL3 -cache $localCache -targetFolder $localDir + $operation += OpSwig3010 -cache $localCache -targetFolder $localDir + $operation += OpProtoBuf310VS15 -cache $localCache -targetFolder $localDir + $operation += OpZlibVS15 -cache $localCache -targetFolder $localDir + if ($ServerLocation) { + $operation += OpProtoBuf310VS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir + $operation += OpZLibVS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir + } + $operation += OpAnaconda3411 -cache $localCache -targetFolder $localDir + $operation += OpAnacondaEnv34 -targetFolder $localDir -repoDir $repositoryRootDir -repoName $reponame + + + #$operation += OpGit2101 -cache $localCache + #$operation += OpGitClone -targetFolder $repositoryRootDir -targetDir $reponame + #$operation += OpSysinternals -cache $localCache -targetFolder $localDir + #$operation += OpOpenCVInternal $ServerLocation -cache $localCache -targetFolder $localDir + #$operation += OpOpenCV31 -cache $localCache -targetFolder $localDir + #$operation += OpCygwin -cache $localCache -targetFolder $localDir + + #$operation += AddOpDisableJITDebug + #$operation += OpTestData "c:\Data\CNTKTestData" "\\storage.ccp.philly.selfhost.corp.microsoft.com\public\CNTKTestData" + #$operation += OpSysinternals -cache $localCache -targetFolder $localDir + + + $operationList = @() + $operationList += (VerifyOperations $operation) + + PreReqOperations $operationList + + if (DisplayAfterVerify $operationList) { + + DownloadOperations $operationList + + ActionOperations $operationList + + #DisplayEnd + } + } + catch { + Write-Host "Exception caught - function main / failure" + Write-Host ($Error[0]).Exception + } +} + +main + +exit 0 + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Action.ps1 b/Scripts/devInstall/Windows/helper/Action.ps1 new file mode 100644 index 000000000000..d8f8c8bce29f --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Action.ps1 @@ -0,0 +1,782 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function ActionOperations( + [parameter(Mandatory=$true)][array] $actionList) +{ + Write-Host "Performing install operations" + + foreach ($item in $actionList) { + if ($item.ActionInfo) { + Write-Host $item.ActionInfo + } + foreach ($actionItem in $item.Action) { + ActionItem $actionItem + } + } + + Write-Host "Install operations finished" + Write-Host +} + +function ActionItem( + [hashtable] $item) +{ + $func = $item["Function"] + + $expr = $func +' $item' + + Write-Verbose "Calling Operation: [$func]" + Invoke-Expression $expr +} + +function InstallExeForPlatform( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $platform = $table["Platform"] + + if (PlatformMatching $platform) { + InstallExe $table + } + return +} + +function InstallExe( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $cmd = $table["Command"] + $param= $table["Param"] + $dir = $table["WorkDir"] + $processWait = $table["ProcessWait"] + $message = $table["message"] + $runAs = GetTableDefaultBool -table $table -entryName "runAs" -defaultValue $true + $maxErrorLevel = GetTableDefaultInt -table $table -entryName "maxErrorLevel" -defaultValue 0 + + if ($message -ne $null) { + Write-Host $message + } + + if ($dir -eq $null) { + DoProcess -doExecute $Execute -command $cmd -param $param -requiresRunAs $runAs -maxErrorLevel $maxErrorLevel -throwOnError $true + } + else { + DoProcess -doExecute $Execute -command $cmd -param $param -requiresRunAs $runAs -workingDir $dir -maxErrorLevel $maxErrorLevel -throwOnError $true + } + + if ( ($processWait -ne $null) -and ($Execute) -and ($false) ) { + do { + start-sleep 20 + $pwait = Get-Process $processWait -ErrorAction SilentlyContinue + } while (-not ($pwait -eq $null)) + } +} + +function InstallYml( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $basePath = $table["BasePath"] + $env= $table["Env"] + $ymlFile = $table["ymlFile"] + + $envsDir = join-path $basePath "envs" + $targetDir = join-path $envsDir $env + + if (test-path -path $targetDir -PathType Container) { + $newTable = @{ Function = "InstallExe"; Command = "$basepath\Scripts\conda.exe"; Param = "env update --file $ymlFile --name $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } + } + else { + $newTable = @{ Function = "InstallExe"; Command = "$basepath\Scripts\conda.exe"; Param = "env create --file $ymlFile --prefix $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } + } + + InstallExe $newTable +} + +function ExecuteApplication( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $appName = $table["AppName"] + $param= $table["Param"] + $dir = $table["WorkDir"] + $appDir = GetTableDefaultString -table $table -entryName "AppDir" -defaultValue [string]::Empty + $usePath = GetTableDefaultBool -table $table -entryName "UseEnvPath" -defaultValue $false + $maxErrorLevel = GetTableDefaultInt -table $table -entryName "maxErrorLevel" -defaultValue 0 + + if (-not $Execute) { + Write-Host "** Running in DEMOMODE - setting Exit Code **: 0" + return + } + $application = ResolveApplicationName $appName $appDir $usePath + if ($application.Length -eq 0) { + throw "ExecuteApplication: Couldn't resolve program [$appName] with location directory [$appDir] and usePath [$usePath]" + } + if ($dir -eq $null) { + DoProcess -doExecute $Execute -command $application -param $param -maxErrorLevel $maxErrorLevel -throwOnError $true + } + else { + DoProcess -doExecute $Execute -command $application -param $param -workingDir $dir -maxErrorLevel $maxErrorLevel -throwOnError $true + } +} + +function InstallWheel( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $BasePath = $table["BasePath"] + $EnvName = $table["EnvName"] + $whl = $table["whl"] + $message = $table["message"] + $whlDirectory = $table["WheelDirectory"] + + Write-Host $message + if (-not $Execute) { + Write-Host "** Running in DEMOMODE - setting Exit Code **: 0" + return + } + $condaExe = Join-Path $BasePath 'Scripts\conda.exe' + $newPaths = Invoke-DosCommand $condaExe (Write-Output ..activate cmd.exe $EnvName) -maxErrorLevel 0 + + $oldPath = $env:PATH + $env:PATH = $newPaths + ';' + $env:PATH + + Invoke-DosCommand pip (Write-Output install $whl) -maxErrorLevel 0 + $env:PATH = $oldPath + return +} + +function InstallMSI( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + + $func = $table["Function"] + $msi = $table["MsiName"] + $dir = $table["MsiDir"] + + $cmd = "c:\Windows\System32\MSIEXEC.EXE" + $param= "/i $dir\$msi /quiet /norestart" + + DoProcess -doExecute $Execute -command $cmd -param "$param" -requiresRunAs $true -maxErrorLevel 0 -throwOnError $true +} + +function MakeDirectory( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $path = $table["Path"] + + if (-not (test-path -path $path)) { + if ($Execute) { + New-Item $path -type directory | Out-Null + } + } + + return +} + +function RobocopyFromLocalCache( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $source = $table["Source"] + $destination = $table["Destination"] + + RobocopySourceDestination $source $destination +} + +function RobocopySourceDestination( + [Parameter(Mandatory = $true)][string] $source, + [Parameter(Mandatory = $true)][string] $destination, + [bool] $copyAdditive=$false) +{ + if (-not (test-path $source -PathType Any)) { + throw SourceDirectory [$source] is missing + } + + + $option = "/NFL /copy:DT /dcopy:D /xj" + if (-not $copyAdditive) { + $option += " /MIR " + } + + $param = "$source $destination $option" + + DoProcess -doExecute $Execute -command $roboCopyCmd -param $param -maxErrorLevel 8 -throwOnError $true + return +} + +function SetEnvironmentVariable( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $name = $table["EnvVar"] + $content = $table["Content"] + $location = "Machine" + + if (-not $Execute) { + return + } + else { + $demoMessage = "" + } + + if ($demoMessage.Length -gt 0) { + Write-Verbose "$demoMessage[$func]: [$name] = [$content] in [$location]" + } + + SetEnvVar -name "$name" -content "$content" + return +} + +function AddToPath( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $dir = $table["Dir"] + $atStart = $table["AtStart"] + $env = $table["env"] + + if ($env.Length -eq 0) { + $env = "PATH" + } + + $pathValue = [environment]::GetEnvironmentVariable($env, "Machine") + if ($pathValue -eq $null) { + $pathValue = "" + } + $pv = $pathValue.ToLower() + $ap = $dir.ToLower() + + if ($pv.Contains("$ap")) { + Write-Verbose "AddToPath - path information already up-to-date" + return + } + + Write-Verbose "Adding [$dir] to environment [$env]" + if ($atStart) { + $pathvalue = "$dir;$pathvalue" + } + else { + $pathvalue = "$pathvalue;$dir" + } + if ($Execute) { + SetEnvVar -name $env -content "$pathvalue" + } + return +} + +function ExtractAllFromZipForPlatform( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $platform = $table["Platform"] + + if (PlatformMatching $platform) { + ExtractAllFromZip $table + } +} + +function ExtractAllFromZip( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $zipFileName = $table["zipFileName"] + $destination = $table["destination"] + $destinationFolder = $table["destinationFolder"] + $zipSubTree = $table["zipSubTree"] + $copyAdditive = GetTableDefaultBool -table $table -entryName "AddToDirectory" -defaultValue $false + + Write-Verbose "ExtractAllFromZip: zipFileName [$zipFileName] destination [$destination] Folder [$destinationFolder]" + + if (-not $Execute) { + return + } + $completeDestination = join-path -Path $destination -ChildPath $destinationFolder + + if (-not (test-path -path $completeDestination)) { + new-item $completeDestination -type directory -Force -ErrorAction Stop | Out-Null + } + if (-not (test-path $zipFileName -PathType Leaf)) { + throw "ExtractAllFromZip: zipFileName [$zipFileName] not found!" + } + + $tempDir = [System.IO.Path]::GetTempFileName(); + + remove-item $tempDir | Out-Null + + $completeTempDestination = join-path -Path $tempDir -ChildPath $destinationFolder + new-item -type directory -path $completeTempDestination -Force -ErrorAction Stop | Out-Null + + + if ($Execute) { + $obj = new-object -com shell.application + $zipFile = $obj.NameSpace($zipFileName) + $destinationNS = $obj.NameSpace($completeTempDestination) + + $destinationNS.CopyHere($zipFile.Items()) + + if ($zipSubTree -ne $null) { + $completeTempDestination = join-path $completeTempDestination $zipSubTree + } + + RobocopySourceDestination $completeTempDestination $completeDestination $copyAdditive + + rm -r tempDir -Force -ErrorAction SilentlyContinue | Out-Null + } + + return +} + +function ExtractAllFromTarGz( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $sourceFile = $table["SourceFile"] + $targzFileName = $table["TargzFileName"] + $destination = $table["destination"] + + Write-Verbose "ExtractAllFromTarGz: targzFileName [$targzFileName] destination [$destination] in TargetFolder [$targetfolder]" + if (-not $Execute) { + return + } + + $appDir = join-path $env:ProgramFiles "git\usr\bin" + $tarApp = "tar.exe" + + if (-not (test-path -path "$appDir\$tarApp" -PathType Leaf)) { + throw "Unpacking the file [$targzFileName] requires extraction utility [$appDir\$tarApp].\n The utility wasn't found" + } + + Copy-Item $sourceFile "$destination\$targzFileName" -ErrorAction SilentlyContinue + + $dosCommand = @" +set path="$appDir";%PATH% & tar.exe -xz --force-local -f "$destination\$targzFileName" -C "$destination" +"@ + + & cmd /c $dosCommand + if ($LASTEXITCODE -gt 0) { + throw "Running [$appDir\tar.exe] Command failed with exit code $LASTEXITCODE" + } + + Remove-Item "$destination\$targzFileName" -ErrorAction SilentlyContinue + + return +} + +function CreateBatch( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $filename = $table["Filename"] + + if (-not $Execute) { + Write-Host "Create-Batch [$filename]:No-Execute flag. No file created" + return + } + + Remove-Item -Path $filename -ErrorAction SilentlyContinue | Out-Null + + $batchScript = @" +@echo off +if /I "%CMDCMDLINE%" neq ""%COMSPEC%" " ( + echo. + echo Please execute this script from inside a regular Windows command prompt. + echo. + exit /b 0 +) +set PATH=$cntkRootDir\cntk;%PATH% +"$AnacondaBasePath\Scripts\activate" "$AnacondaBasePath\envs\cntk-py34" +"@ + + add-content -Path $filename -Encoding Ascii -Value $batchScript +} + +function SetRegistryKey( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["key"] + $elevated = $table["Elevated"] + + if ($Execute) { + $result = Test-Path -Path $key + + if (-not $result) { + Write-Verbose "[$func]: [$key] will be created" + if ($elevated) { + $commandString = "& { new-item -Path '$key' }" + RunPowershellCommand -command "$commandString" -elevated $true -maxErrorLevel 0 + } + else { + new-item -Path $key + } + } + } + return +} + +function SetRegistryKeyNameData( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["key"] + $regName = $table["RegName"] + $data = $table["data"] + $dataType = $table["dataType"] + $elevated = GetTableDefaultBool -table $table -entryName "Elevated" -defaultValue $true + + if ($Execute) { + $tab = @{Function = "SetRegistryKey"; Key=$key; Elevated=$elevated} + SetRegistryKey $tab + + Write-Verbose "[$func]: [$key]:[$regName] will be set to [$dataType]:[$data]" + + $commandString = "& { set-itemproperty -path '$key' -name '$regName' -Type $dataType -Value $data }" + RunPowershellCommand -command "$commandString" -elevated $elevated -maxErrorLevel 0 + + + #if ($elevated) { + # $commandString = "& { set-itemproperty -path '$key' -name '$regName' -Type $dataType -Value $data }" + # RunPowershellCommand -command "$commandString" -elevated $true -maxErrorLevel 0 + #} + #else { + # set-itemproperty -path '$key' -name '$regName' -Type $dataType -Value $data + #} + } +} + +function CreateBuildProtobufBatch( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $filename = $table["FileName"] + $sourceDir = $table["SourceDir"] + $targetDir = $table["TargetDir"] + + if ($Execute) { + Remove-Item -Path $filename -ErrorAction SilentlyContinue | Out-Null + + $batchScript = GetBatchBuildProtoBuf $sourceDir $targetDir + + add-content -Path $filename -Encoding Ascii -Value $batchScript + } +} + +function CreateBuildZlibBatch( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $filename = $table["FileName"] + $zlibSourceDir = $table["zlibSourceDir"] + $libzipSourceDir = $table["libzipSourceDir"] + $targetDir = $table["TargetDir"] + + if ($Execute) { + Remove-Item -Path $filename -ErrorAction SilentlyContinue | Out-Null + + $batchScript = GetBatchBuildZlibBuf $zlibSourceDir $libzipSourceDir $targetDir + + add-content -Path $filename -Encoding Ascii -Value $batchScript + } +} + +function DoProcess( + [bool] $doExecute = $false, + [string] $command, + [string] $param, + [string] $workingDir = "", + [boolean] $requiresRunAs = $false, + [int] $maxErrorLevel = 0, + [bool] $throwOnError = $true) +{ + Write-Verbose "start-process [$command] with [$param]" + + if (-not $DoExecute) { + Write-Host "DEMOMODE - setting Exit Code: 0" + return + } + + if ($workingDir.Length -eq 0) { + if ($requiresRunAs) { + $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -Verb runas + } + else { + $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru + } + + } + else { + if ($requiresRunAs) { + $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -Verb runas -WorkingDirectory "$workingDir" + } + else { + $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -WorkingDirectory "$workingDir" + } + } + + $eCode = ($process.ExitCode) + + if (-not $throwOnError) { + if ($ecode -gt $maxErrorLevel) { + Write-Verbose "Running [start-process $commandString $param] failed with exit code [$ecode]" + return + } + return + } + + if ($ecode -gt $maxErrorLevel) { + throw "Running [start-process $commandString $param] failed with exit code [$ecode]" + } + return +} + +function SetEnvVar( + [Parameter(Mandatory=$true)][string] $name, + [Parameter(Mandatory=$true)][string] $content, + [string] $location = "Machine") +{ + Write-Verbose "SetEnvVar [$name] with [$content]" + + if ($Execute) { + $commandString = "& { [environment]::SetEnvironmentVariable('"+$name+"', '"+$content+"', '"+$location+"') }" + RunPowershellCommand -command "$commandString" -elevated $true -maxErrorLevel 0 + } +} + +function RunPowershellCommand( + [string] $commandString, + [boolean] $elevated, + [int] $maxErrorLevel) +{ + $commandBytes = [System.Text.Encoding]::Unicode.GetBytes($commandString) + $encodedCommand = [Convert]::ToBase64String($commandBytes) + $commandLine = "-NoProfile -EncodedCommand $encodedCommand" + + if ($elevated) { + $process = Start-Process -PassThru -FilePath powershell.exe -ArgumentList $commandLine -wait -verb runas + } + else { + $process = Start-Process -PassThru -FilePath powershell.exe -ArgumentList $commandLine -wait + } + $eCode = ($process.ExitCode) + if ($ecode -gt $maxErrorLevel) { + throw "Running 'powershell.exe $commandString' failed with exit code [$ecode]" + } + return +} + +function Invoke-DosCommand { + [CmdletBinding()] + Param( + [ValidateScript({ Get-Command $_ })] + [string] $Command, + [string[]] $Argument, + [string] [ValidateScript({ Test-Path -PathType Container $_ })] $WorkingDirectory, + [int] $maxErrorLevel, + [switch] $SuppressOutput + ) + Write-Verbose "Running '$Command $Argument'" + if ($WorkingDirectory) { + Push-Location $WorkingDirectory -ErrorAction Stop + } + if ($SuppressOutput) { + $null = & $Command $Argument 2>&1 + } else { + & $Command $Argument + } + if ($WorkingDirectory) { + Pop-Location + } + if ($LASTEXITCODE -gt $maxErrorLevel) { + throw "Running DOS Command '$Command $Argument' failed with exit code $LASTEXITCODE" + } +} + +function ResolveApplicationName( + [string] $name, + [string] $directory, + [bool] $usePath) +{ + $application = "" + + if ($directory.Length -gt 0) { + $application = CallGetCommand (join-path $directory $name) + } + if ($application.Length -eq 0) { + if ($usePath) { + # we are at this point if we are supposed to check in the path environment for a match and + # $directory was empty or we couldn't find it in the $directory + + $application = CallGetCommand $name + } + } + # application will be an empty string if we couldn't resolve the name, otherwise we can execute $application + + return $application +} + +function CallGetCommand( + [string] $application) +{ + try { + get-command $application -CommandType Application -ErrorAction Stop | Out-Null + return $application + } + catch { + # the application can't be found, so return empty string + return "" + } +} + +function GetBatchBuildProtoBuf( + [string] $sourceDir, + [string] $targetDir) +{ + $batchscript = @" +@SET VCDIRECTORY=C:\Program Files (x86)\Microsoft Visual Studio 14.0 +@SET SOURCEDIR=$sourceDir +@SET TARGETDIR=$targetDir + +@echo. +@echo This will build Protobuf-3.1.0 +@echo ------------------------------ +@echo The configured settings for the batch file: +@echo Visual Studio directory: %VCDIRECTORY% +@echo Protobuf source directory: %SOURCEDIR% +@echo Protobuf target directory: %TARGETDIR% +@echo. +@echo. +@echo Please edit the batch file if this doesn't match your directory layout! +@echo. + +@pause + +@call "%VCDIRECTORY%\VC\vcvarsall.bat" amd64 + +@pushd %SOURCEDIR% +@cd cmake +@md build && cd build + +@md debug && cd debug +@cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" ..\.. +@nmake +@nmake install +@cd .. + +@md release && cd release +@cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" ..\.. +@nmake +@nmake install +@cd .. + +@popd + +setx PROTOBUF_PATH %TARGETDIR% +"@ + + return $batchscript +} + +function GetBatchBuildZlibBuf( + [string] $zlibSourceDir, + [string] $libzipSourceDir, + [string] $targetDir) +{ + $batchscript = @" +@SET VCDIRECTORY=C:\Program Files (x86)\Microsoft Visual Studio 14.0 +@SET LIBZIPSOURCEDIR=$libzipSourceDir +@SET ZLIBSOURCEDIR=$zlibSourceDir +@SET TARGETDIR=$targetDir +@SET CMAKEGEN="Visual Studio 14 2015 Win64" + +@echo. +@echo This will build ZLib using Visual Studio 2015 +@echo --------------------------------------------- +@echo The configured settings for the batch file: +@echo Visual Studio directory: %VCDIRECTORY% +@echo CMake Generator: %CMAKEGEN% +@echo LibZip source directory: %LIBZIPSOURCEDIR% +@echo Zlib source directory: %ZLIBSOURCEDIR% +@echo Zlib-VS15 target directory: %TARGETDIR% +@echo. +@echo. +@echo Please edit the batch file if this doesn't match your directory layout! +@echo. + +@pause + +@call "%VCDIRECTORY%\VC\vcvarsall.bat" amd64 + +@pushd %ZLIBSOURCEDIR% +@mkdir build +@cd build +@cmake .. -G%CMAKEGEN% -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" +@msbuild /P:Configuration=Release INSTALL.vcxproj +@popd + +@pushd %LIBZIPSOURCEDIR% +@md build +@cd build +@cmake .. -G%CMAKEGEN% -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" +@msbuild libzip.sln /t:zip /P:Configuration=Release +@cmake -DBUILD_TYPE=Release -P cmake_install.cmake +@popd + +setx ZLIB_PATH %TARGETDIR% +"@ + + return $batchscript +} + + + +function GetCygwinBashScript +{ + $batchscript = @" +easy_install-2.7 pip +pip install six +pip install pytest +"@ + + return $batchscript +} + + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Common.ps1 b/Scripts/devInstall/Windows/helper/Common.ps1 new file mode 100644 index 000000000000..348b16c95ab8 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Common.ps1 @@ -0,0 +1,42 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# + +function GetTableDefaultBool( + [hashtable] $table, + [string] $entryName, + [bool] $defaultValue +) +{ + if ($table[$entryName] -eq $null) { + return $defaultValue + } + return $table[$entryName] +} + +function GetTableDefaultString( + [hashtable] $table, + [string] $entryName, + [string] $defaultValue +) +{ + if ($table[$entryName] -eq $null) { + return $defaultValue + } + return $table[$entryName] +} + +function GetTableDefaultInt( + [hashtable] $table, + [string] $entryName, + [int] $defaultValue +) +{ + if ($table[$entryName] -eq $null) { + return $defaultValue + } + return $table[$entryName] +} + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Display.ps1 b/Scripts/devInstall/Windows/helper/Display.ps1 new file mode 100644 index 000000000000..49f2c5c93a04 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Display.ps1 @@ -0,0 +1,149 @@ +function FunctionIntro( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + $table | Out-String | Write-Verbose +} + +function GetKey( + [string] $validChar +) +{ + do { + $key = Read-Host + } until ($key -match $validChar) + + return $key +} + +function DisplayStartMessage +{ +" + +This script will setup the CNTK v2 Development Environment on your machine. +More help is given by calling get-help .\devInstall.ps1 + +The script will analyse your machine and will determine which components are required. +The required components will be downloaded into [$localCache] +Repeated operation of this script will reuse already downloaded components. +" +} + +function DisplayVersionWarningMessage( + [string] $version) +{ +" +You are executing this script from Powershell Version $version. +We recommend that you execute the script from Powershell Version 4 or later. You can install Powershell Version 4 from: + https://www.microsoft.com/en-us/download/details.aspx?id=40855 +" +} + +function DisplayWarningNoExecuteMessage +{ +" +The parameter '-Execute' hasn't be supplied to the script. +The script will execute withouth making any actual changes to the machine! +" +} + +function DisplayStartContinueMessage +{ +" +1 - I agree and want to continue +Q - Quit the installation process +" +} + +function CheckPowershellVersion +{ + $psVersion = $PSVersionTable.PSVersion.Major + if ($psVersion -ge 4) { + return $true + } + + Write-Warning $(DisplayVersionWarningMessage $psVersion) + if ($psVersion -eq 3) { + return $true + } + return $false +} + +function CheckOSVersion +{ + $runningOn = (Get-WmiObject -class Win32_OperatingSystem).Caption + $isMatching = ($runningOn -match "^Microsoft Windows (8\.1|10|Server 2012 R2|Server 2016)") + + if (-not $isMatching) { + Write-Warning " +You are running the this install script on [$runningOn]. +The Microsoft Cognitive Toolkit is designed and tested on Windows 8.1, Windows 10, +Windows Server 2012 R2, and Windows Server 2016. +" + } +} + +function DisplayStart() +{ + Write-Host $(DisplayStartMessage) + + if (-not (CheckPowershellVersion)) { + return $false + } + + CheckOSVersion + + if (-not $Execute) { + Write-Warning $(DisplayWarningNoExecuteMessage) + } + + Write-Host $(DisplayStartContinueMessage) + $choice = GetKey '^[1qQ]+$' + + if ($choice -contains "1") { + return $true + } + + return $false +} + + +Function DisplayEnd() +{ + +} + +function DisplayAfterVerify( + [array] $list = @()) +{ + Write-Host + + if ($list.Count -gt 0) { + Write-Host "The following operations will be performed:" + + foreach ($item in $list) { + $info = $item.ActionInfo + Write-Host " * $info" + } + if (-not $Execute) { + Write-Warning $(DisplayWarningNoExecuteMessage) + } + + Write-Host + Write-Host "Do you want to continue? (y/n)" + + $choice = GetKey '^[yYnN]+$' + + if ($choice -contains "y") { + return $true + } + } + else { + Write-Host "No additional installation required" + Write-Host + + } + return $false +} + +# vim:set expandtab shiftwidth=2 tabstop=2: diff --git a/Scripts/devInstall/Windows/helper/Download.ps1 b/Scripts/devInstall/Windows/helper/Download.ps1 new file mode 100644 index 000000000000..6bc4c6771187 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Download.ps1 @@ -0,0 +1,298 @@ +function DownloadOperations( + [parameter(Mandatory=$true)][array] $downloadList) +{ + Write-Host "Performing download operations" + + foreach ($item in $downloadList) { + foreach ($downloadItem in $item.Download) { + DownloadItem $downloadItem + } + } + + Write-Host "Download operations finished" + Write-Host +} + + +function DownloadItem( + [hashtable] $item +) +{ + $func = $item["Function"] + + $expr = $func +' $item' + + Write-Verbose "Calling Operation: [$func]" + $result = Invoke-Expression $expr + + return +} + +function DownloadForPlatform( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $platform = $table["Platform"] + + if (PlatformMatching $platform) { + Download $table + } +} + +function Download( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $source = $table["Source"] + $method = GetTableDefaultString -table $table -entryName "Method" -defaultValue "WebRequest" + $destination = $table["Destination"] + $ExpectedSize = GetTableDefaultInt -table $table -entryName "ExpectedSize" -defaultValue 0 + + if (test-path $destination -PathType Leaf) { + Write-Host File [$destination] already exists + return + } + + if ($method -eq "WebRequest") { + DownloadFileWebRequest -SourceFile $source -OutFile $destination -ExpectedSize $ExpectedSize + } + else { + DownloadFile -SourceFile $source -OutFile $destination + } + return +} + +function LocalCopyFile( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $source = $table["Source"] + $destination = $table["Destination"] + + if (-not $Execute) { + Write-Host "$message ** Running in DEMOMODE - no download performed" + return $true + } + if (test-path $destination -PathType Leaf) { + Write-Host File [$destination] already exists + return + } + if (-not (test-path $source -PathType Leaf)) { + throw "Sourcefile [$source] is missing" + } + + Write-Host Copying [$source] to local disk ... + new-item $destination -type File -Force -ErrorAction SilentlyContinue + copy-Item $source $destination -Force -ErrorAction SilentlyContinue + return +} + +function RobocopyFromServer( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + + $source = $table["Source"] + $destination = $table["Destination"] + + $source = $LocalServer+"\$source" + + if (-not (test-path $source)) { + throw "SourceDirectory [$source] is missing" + } + + $option = "/MIR /NFL /copy:DT /dcopy:D /xj" + + $param = "$source $destination $option" + + DoProcess -command $roboCopyCmd -param "$param" -IgnoreNonZeroExitCode $true +} + +function NotImplemented( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + throw "Call to function 'NotImplemented' " +} + +function DownloadAndExtract( + [string] $tPath, + [string] $sAddress, + [string] $fileName, + [string] $targetPathRoot +){ + $outFileName = Join-Path $tPath $fileName + + DownloadFile -SourceFile $sAddress ` + -OutFile $outFileName ` + -tempFileName $fileName + + Write-Host Extracting into targetpath + Write-Host + + ExtractAllFromZip $outFileName $targetPathRoot +} + + +function DownloadFileWebRequest ( + [string] $SourceFile, + [string] $OutFile, + [int] $ExpectedSize) +{ + Write-Host "Downloading [$SourceFile], please be patient...." + if (-not $Execute) { + Write-Host "$message ** Running in DEMOMODE - no download performed" + return $true + } + + if (test-path -path $outFile) { + Write-Host "File [$outFile] already exists" + return $true + } + + $TempFile = [System.IO.Path]::GetTempFileName() + try { + $response = Invoke-WebRequest -Uri $SourceFile -OutFile $TempFile -TimeoutSec 120 + } + catch { + $errorCode = $_.Exception.Response.StatusCode.Value__ + + Remove-Item -path $TempFile -ErrorAction SilentlyContinue + throw "Download $SourceFile Failed! WebRequest reported errorCode: [$errorCode]" + } + + # we now have the temporary file + if (($ExpectedSize -gt 0) -and (Test-Path $TempFile)) { + $fileSize = (Get-Item $TempFile).length + if (-not ($fileSize -eq $ExpectedSize)) { + Remove-Item -path $TempFile -ErrorAction SilentlyContinue + throw "Size of downloaded file [$fileSize] not matching expected filesize [$ExpectedSize] for $sourceFile" + } + } + + new-item $outFile -type File -Force -ErrorAction SilentlyContinue + move-Item $TempFile $OutFile -Force -ErrorAction SilentlyContinue + + if (-not (Test-Path -Path $OutFile)) { + throw "Download $SourceFile Failed!" + } + # we have a file with our expected filename, we are in good shape and ready to report success + # in case the above rename failed, but the target file exist, we clean up a possible dangling TempFile + # no need to check if this file is really there. we don't care if it succeeds or not + Remove-Item -path $TempFile -ErrorAction SilentlyContinue +} + +function DownloadFile( + [string] $SourceFile, + [string] $OutFile, + [int] $timeout = 600, + [int] $maxtry = 5) +{ + # the scriptblock to invoke a web-request + $sb ={ + param([string]$uri,[string]$outfile) + (New-Object System.Net.WebClient).DownloadFile($uri,$outfile) + } + #--------------- + + $startTime = Get-Date + Write-Host "Downloading [$SourceFile], please be patient...." + if (-not $Execute) { + Write-Host "$message ** Running in DEMOMODE - no download performed" + return + } + + $TempFile = [System.IO.Path]::GetTempFileName() + + for ($count=1; $count -le $maxtry; $count +=1) { + if ($count -gt 1) { + Write-Host "Iteration [$count] of [$maxtry]" + } + + if ($count -gt 1) { + start-sleep -Seconds 5 + } + if (Test-Path -Path $TempFile) { + # the file we temporary use as a download cache could exist + # if it does, we remove it and terminate if this fails + Remove-Item -path $TempFile -ErrorAction Stop + } + + if (test-path -path $outFile) { + Write-Host "File [$outFile] already exists" + return + } + + $job = start-job -scriptblock $sb -ArgumentList $sourceFile, $TempFile + Wait-Job $job -Timeout $timeout + + $jState = $job.State.ToUpper() + $jStart = $job.PSBeginTime + $jEnd = $job.PSEndTime + $jError = $job.ChildJobs[0].Error + $current = Get-Date + + switch ($jState) { + "COMPLETED" { + if ($jError.Count -eq 0) { + Write-Verbose "End binary download!" + + Remove-Job $job -force -ErrorAction SilentlyContinue + + # we now have the temporary file, we need to rename it + new-item $outFile -type File -Force -ErrorAction SilentlyContinue + move-Item $TempFile $OutFile -Force -ErrorAction SilentlyContinue + + if (Test-Path -Path $OutFile) { + # we have a file with our expected filename, we are in good shape and ready to report success + # in case the above rename failed, but the target file exist, we clean up a possible dangling TempFile + # no need to check if this file is really there. we don't care if it succeeds or not + Remove-Item -path $TempFile -ErrorAction SilentlyContinue + + return + } + + # we got here because we finished the job, but some operation failed (i.e. the rename above. we can just try again) + Write-Verbose "Job completed but rename operation failed, retrying..." + continue + } + + Write-Host "Job Completed with Error: [$jStart] to [$current]" + Write-Host $jError + } + "RUNNING" { + Write-Host "Job Timeout: [$jStart] to [$current]" + } + "FAILED" { + $current = Get-Date + Write-Host "Job Failed: [$jStart] to [$current]" + Write-Host "Error: $jError" + } + default { + Write-Host "Job State: [$Error] - [$jStart] to [$current]" + } + } + Remove-Job $job -force -ErrorAction SilentlyContinue + } + + throw "Download $SourceFile Failed!" +} + +function PlatformMatching( + [string] $regExprPlatform) +{ + $runningOn = ((Get-WmiObject -class Win32_OperatingSystem).Caption).ToUpper() + $isMatching = ($runningOn -match $regExprPlatform) + + Write-Verbose "Function [PlatformMatching]: $runningOn -match on platform [$regExprPlatform] = [$isMatching]" + return $isMatching +} + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Operations.ps1 b/Scripts/devInstall/Windows/helper/Operations.ps1 new file mode 100644 index 000000000000..f2980af245db --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Operations.ps1 @@ -0,0 +1,662 @@ +function OpAnaconda3411( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Anaconda3-4.1.1" + $prodFile = "Anaconda3-4.1.1-Windows-x86_64.exe" + $prodSubDir = "Anaconda3-4.1.1-Windows-x86_64" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://repo.continuum.io/archive/Anaconda3-4.1.1-Windows-x86_64.exe" + $downloadSize = 370055720 + + @( @{ShortName = "ANA3-411"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath; } ); + Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/InstallationType=JustMe /AddToPath=0 /RegisterPython=0 /S /D=$targetPath"; runAs=$false; Message = ".... This will take some time. Please be patient ...." } ); + } ) +} + +function OpAnacondaEnv34( + [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $repoDir, + [parameter(Mandatory=$true)][string] $reponame) +{ + $prodName = "Python 3.4 Environment" + + $prodSubDir = "Anaconda3-4.1.1-Windows-x86_64" + $targetPath = join-path $targetFolder $prodSubDir + $envName = "cntkdev-py34" + $envDir = "envs\$envName" + $envVar = "CNTK_PY34_PATH"; + $envValue = join-path $targetPath $envDir + + $ymlDirectory = join-path $repoDir $repoName + $ymlDirectory = join-path $ymlDirectory "scripts\install\windows" + + @{ ShortName = "PYENV34"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Creating $prodName"; + Verification = @( @{Function = "VerifyRunAlways" } ); + Action = @( @{Function = "InstallYml"; BasePath = $targetPath; Env = $envName; ymlFile= "$ymlDirectory\conda-windows-cntk-py34-environment.yml" }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ) + } +} + +function OpBoost160( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Boost 1.60.0" + $prodFile = "boost_1_60_0-msvc-12.0-64.exe" + $prodSubDir = "boost_1_60_0" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://sourceforge.net/projects/boost/files/boost-binaries/1.60.0/boost_1_60_0-msvc-12.0-64.exe/download" + + @( @{Name = $prodName; ShortName = "BOOST160"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "BOOST_INCLUDE_PATH"; Content = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "BOOST_LIB_PATH"; Content = "$targetPath\lib64-msvc-12.0" } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = "BOOST_INCLUDE_PATH"; Content = "$targetPath" }, + @{Function = "SetEnvironmentVariable"; EnvVar = "BOOST_LIB_PATH"; Content = "$targetPath\lib64-msvc-12.0" }, + @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=$targetPath /SP- /SILENT /NORESTART"; runAs=$false } ); + } ) +} + +function OpBoost160VS15( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Boost 1.60.0" + $prodFile = "boost_1_60_0-msvc-14.0-64.exe" + $prodSubDir = "boost_1_60_0-msvc-14.0" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://sourceforge.net/projects/boost/files/boost-binaries/1.60.0/boost_1_60_0-msvc-14.0-64.exe/download" + $envVar = "BOOST_INCLUDE_PATH" + $envVarLib = "BOOST_LIB_PATH" + $envContentLib = "$targetPath\lib64-msvc-14.0" + + @( @{Name = $prodName; ShortName = "BOOST160VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVarLib; Content = $envContentLib } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $targetPath }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVarLib; Content = $envContentLib }, + @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=$targetPath /SP- /SILENT /NORESTART"; runAs=$false } ); + } ) +} + +function OpCMake362( + [parameter(Mandatory=$true)][string] $cache) +{ + $prodName = "CMake 3.6.2" + $targetPath = join-path $env:ProgramFiles "cmake\bin" + $cmakeName = "cmake-3.6.2-win64-x64.msi" + $downloadSource = "https://cmake.org/files/v3.6/cmake-3.6.2-win64-x64.msi" + + @( @{ShortName = "CMake362"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyWinProductExists"; Match = "^CMake$"; Version = "3.6.2" } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$cmakeName" } ); + Action = @( @{Function = "InstallMsi"; MsiName = "$cmakeName" ; MsiDir = "$cache" } , + @{Function = "AddToPath"; Dir = "$targetPath" } ); + } ) +} + +function OpCNTKMKL2 + ([parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "CNTK Custom MKL Version 2" + $prodFile = "CNTKCustomMKL-Windows-2.zip" + $prodSubDir = "CNTKCustomMKL" + $targetPath = join-path $targetFolder $prodSubDir + $targetPathCurrenVersion = join-path $targetPath "2" + $envVar = "CNTK_MKL_PATH"; + $envValue = $targetPath + $downloadSource = "https://www.cntk.ai/mkl/$prodFile"; + + @( @{ShortName = "CNTKMKL2"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPathCurrenVersion"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPathCurrenVersion }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $localDir; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); + } ) +} + +function OpCNTKMKL3 + ([parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "CNTK Custom MKL Version 3" + $prodFile = "CNTKCustomMKL-Windows-3.zip" + $prodSubDir = "CNTKCustomMKL" + $targetPath = join-path $targetFolder $prodSubDir + $targetPathCurrenVersion = join-path $targetPath "3" + $envVar = "CNTK_MKL_PATH"; + $envValue = $targetPath + $downloadSource = "https://www.cntk.ai/mkl/$prodFile"; + + @( @{ShortName = "CNTKMKL3"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPathCurrenVersion"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPathCurrenVersion }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $localDir; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); + } ) +} + +function OpCygwin + ([parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "CygWin64" + $prodFile = "cygwinSetup_64.exe" + $prodSubDir = "cygwin64" + $targetPath = join-path $targetFolder $prodSubDir + $regkey = "Registry::HKEY_CURRENT_USER\SOFTWARE\Cygwin" + $downloadSource = "https://www.cygwin.com/setup-x86_64.exe" + $installParam = "--quiet-mode --no-admin --site http://mirrors.kernel.org/sourceware/cygwin/ --root $targetPath --local-package-dir $cache --packages python,python-yaml,python-numpy,diffutils" + $bFileName = "cygwinpip.bash" + $bashFileName = join-path $targetFolder $bFileName + $bashParmFile = Join-Path "\cygdrive\c" (split-path $bashFileName -NoQualifier) + $bashParmFile = $bashParmFile.replace("\","/") + $bashParam = "-l -c $bashParmFile" + + @( @{ShortName = "CYGWIN64"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, + @{Function = "VerifyRegistryKey"; Key = $regKey } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = $installParam; runAs=$false; WorkDir = $targetFolder}, + @{Function = "AddToPath"; Dir = "$targetPath\bin"; AtStart = $false }, + @{Function = "CreateCygwinBashScript"; FileName = $bashFileName }, + @{Function = "InstallExe"; Command = "$targetPath\bin\bash.exe"; Param = $bashParam; runAs=$false; WorkDir = $targetPath } ); + } ) +} + +function AddOpDisableJITDebug +{ + @( @{Name="Visual Studio Disable Just-In-Time Debugging"; ShortName = "JITVS2013"; VerifyInfo = "Checking for JIT Debugging registry keys"; ActionInfo = "Removing registry keys to disable JIT debugging, advisable for Jenkins machines"; + Verification = @( @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug"; RegName = "Debugger"; }, + @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework"; RegName = "DbgManagedDebugger" }, + @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows NT\CurrentVersion\AeDebug"; RegName = "Debugger" }, + @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\.NETFramework"; RegName = "DbgManagedDebugger" } ); + Action = @( @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug"; Elevated = $true; RegName = "Debugger" }, + @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework"; Elevated = $true; RegName = "DbgManagedDebugger" }, + @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows NT\CurrentVersion\AeDebug"; Elevated = $true; RegName = "Debugger" }, + @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\.NETFramework"; Elevated = $true; RegName = "DbgManagedDebugger" } ) + } ) + } + +function OpGit2101( + [parameter(Mandatory=$true)][string] $cache) +{ + $prodName = "Git" + $targetPath = join-path $env:ProgramFiles "Git\bin" + $prodFile = "Git-2.10.1-64-bit.exe" + $downloadSource = "https://github.com/git-for-windows/git/releases/download/v2.10.1.windows.1/$prodFile" + + @( @{ShortName = "GIT"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\GitForWindows"; RegName = "CurrentVersion"; } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/SP- /SILENT /NORESTART"}, + @{Function = "AddToPath"; Dir = $targetPath; AtStart = $true; } ) + } ) +} + +function OpGitClone( + [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $targetDir, + [string] $repoTag = "master") +{ + $targetPath = join-path $targetFolder $targetDir + $downloadSource = "https://github.com/Microsoft/CNTK/"; + $appDir = join-path $env:ProgramFiles "Git\bin" + + @( @{Name = "Clone CNTK from Github"; ShortName = "CNTKCLONE"; VerifyInfo = "Checking for CNTK-Clone target directory $targetPath"; ActionInfo = "Cloneing CNTK from Github repository"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Action = @( @{Function = "MakeDirectory"; Path = $targetFolder }, + @{Function = "ExecuteApplication"; AppName = "git.exe"; Param = "clone --branch $repoTag --recursive https://github.com/Microsoft/CNTK/"; AppDir = $appDir; UseEnvPath = $true; WorkDir = $targetFolder } ) + } ) +} + +function OpMSMPI70([parameter( + Mandatory=$true)][string] $cache) +{ + $remoteFilename = "MSMpiSetup.exe" + $localFilename = "MSMpiSetup70.exe" + $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; + + @( @{Name = "MSMPI Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70"; ActionInfo = "Installing MSMPI 70"; + Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$localFilename" } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$localFilename" ; Param = "/unattend" } ) + } ) +} + +function OpMSMPI70SDK( + [parameter(Mandatory=$true)][string] $cache) +{ + $remoteFilename = "msmpisdk.msi" + $localFilename = "msmpisdk70.msi" + $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; + + @( @{Name = "MSMPI SDK70 Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70 SDK"; ActionInfo = "Install MSMPI 70 SDK"; + Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI SDK \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); + #Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft MPI SDK \(\d+\."; Compare = "^Microsoft MPI SDK \(7\.0\.12437\.6\)"; MatchExact = $false } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$localFilename" } ); + Action = @( @{Function = "InstallMsi"; MsiName = "$localFilename" ; MsiDir = "$cache" } ) + } ) +} + +function OpNvidiaCub141( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "NVidia CUB 1.4.1" + $prodFile = "cub-1.4.1.zip" + $prodSubDir = "cub-1.4.1" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "CUB_PATH"; + $envValue = $targetPath + $downloadSource = "https://codeload.github.com/NVlabs/cub/zip/1.4.1"; + + @( @{ShortName = "CUB141"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = "$targetFolder"; destinationFolder = $prodSubDir; zipSubTree= $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); + } ) +} + +function OpNVidiaCuda75( + [parameter(Mandatory=$true)][string] $cache) +{ + $cudaDownload = "http://developer.download.nvidia.com/compute/cuda/7.5/Prod/local_installers" + $programPath = join-path $env:ProgramFiles "NVIDIA GPU Computing Toolkit\CUDA\v7.5" + + + @( @{Name="NVidia CUDA 7.5"; ShortName = "CUDA75"; VerifyInfo = "Checking for installed NVidia Cuda 75"; ActionInfo = "Installing CUDA 7.5"; + Verification = @( @{Function = "VerifyDirectory"; Path = $programPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "CUDA_PATH_V7_5"; Content = $programPath } ); + Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Source = "$cudaDownload/cuda_7.5.18_windows.exe"; Destination = "$cache\cuda_7.5.18_windows.exe" }, + @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 10"; Source = "$cudaDownload/cuda_7.5.18_win10.exe"; Destination = "$cache\cuda_7.5.18_win10.exe" } ); + Action = @( @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Command = "$cache\cuda_7.5.18_windows.exe"; Param = "-s CUDAToolkit_7.5 CUDAVisualStudioIntegration_7.5 GDK"; Message = ".... This will take some time. Please be patient ...." } , + @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows 10"; Command = "$cache\cuda_7.5.18_win10.exe"; Param = "-s CUDAToolkit_7.5 CUDAVisualStudioIntegration_7.5 GDK"; Message = ".... This will take some time. Please be patient ...." } ); + }) +} + +function OpNVidiaCuda80( + [parameter(Mandatory=$true)][string] $cache) +{ + $cudaDownload = "http://developer.download.nvidia.com/compute/cuda/8.0/secure/prod/local_installers" + $cudaFile = "cuda_8.0.44_windows.exe" + $cudaFileWin10 = "cuda_8.0.44_win10.exe" + $programPath = join-path $env:ProgramFiles "NVIDIA GPU Computing Toolkit\CUDA\v8.0" + $cudaInstallParam = "-s" + + @( @{Name="NVidia CUDA 8.0"; ShortName = "CUDA80"; VerifyInfo = "Checking for installed NVidia Cuda 80"; ActionInfo = "Installing CUDA 8.0"; + Verification = @( @{Function = "VerifyDirectory"; Path = $programPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "CUDA_PATH_V8_0"; Content = $programPath } ); + Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Source = "$cudaDownload/$cudaFile"; Destination = "$cache\$cudaFile" }, + @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 10"; Source = "$cudaDownload/$cudaFileWin10"; Destination = "$cache\$cudaFileWin10" } ); + Action = @( @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Command = "$cache\$cudaFile"; Param = $cudaInstallParam } , + @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows 10"; Command = "$cache\$cudaFileWin10"; Param = $cudaInstallParam } ); + }) +} + +function OpNVidiaCudnn5175( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) + +{ + $prodName = "NVidia CUDNN 5.1 for CUDA 7.5" + $cudnnWin7 = "cudnn-7.5-windows7-x64-v5.1.zip" + $cudnnWin10 = "cudnn-7.5-windows10-x64-v5.1.zip" + + $prodSubDir = "cudnn-7.5-v5.1" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "CUDNN_PATH" + $envValue = join-path $targetPath "cuda" + $downloadSource = "http://developer.download.nvidia.com/compute/redist/cudnn/v5.1" + + @( @{ShortName = "CUDNN5175"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 7"; Source = "$downloadSource/$cudnnWin7"; Destination = "$cache\$cudnnWin7" }, + @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; Source = "$downloadSource/$cudnnWin10"; Destination = "$cache\$cudnnWin10" } ); + Action = @( @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows 7"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, + @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + }) +} + +function OpNVidiaCudnn5180( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "NVidia CUDNN 5.1 for CUDA 8.0" + $cudnnWin7 = "cudnn-8.0-windows7-x64-v5.1.zip" + $cudnnWin10 = "cudnn-8.0-windows10-x64-v5.1.zip" + + $prodSubDir = "cudnn-8.0-v5.1" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "CUDNN_PATH" + $envValue = join-path $targetPath "cuda" + $downloadSource = "http://developer.download.nvidia.com/compute/redist/cudnn/v5.1" + + @( @{ShortName = "CUDNN5180"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyDirectory"; Path = $envValue }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{Function = "DownloadForPlatform"; Platform = "^Microsoft Windows 7"; Source = "$downloadSource/$cudnnWin7"; Destination = "$cache\$cudnnWin7" }, + @{Function = "DownloadForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; Source = "$downloadSource/$cudnnWin10"; Destination = "$cache\$cudnnWin10" } ); + Action = @( @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows 7"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, + @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + }) +} + +function OpOpenCV31( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "OpenCV-3.1" + $prodFile = "opencv-3.1.0.exe" + $prodSubDir = "OpenCV310" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "OPENCV_PATH_V31"; + $envValue = "$targetPath\build" + $downloadSource = "https://netcologne.dl.sourceforge.net/project/opencvlibrary/opencv-win/3.1.0/opencv-3.1.0.exe" + + @( @{ShortName = "OPENCV310"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Please perform a manual installation of $prodName from $cache"; + Verification = @( @{Function = "VerifyFile"; Path = "$cache\$prodFile" } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + } ) +} + +function OpOpenCVInternal( + [parameter(Mandatory=$true)][string] $server, + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "OpenCV-3.1" + $prodFile = "opencv-3.1.0.zip" + $prodSubDir = "Opencv3.1.0" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "OPENCV_PATH_V31"; + $envValue = "$targetPath\build" + $downloadSource = "$server\$prodFile" + + @( @{ShortName = "OPENCV310"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "LocalCopyFile"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = "$targetFolder"; destinationFolder = $prodSubDir; zipSubTree= $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); + } ) +} + +function OpProtoBuf310VS15( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + # unzip protobuf source in $protoSourceDir = $targetfolder\src\$prodsubdir + # create batch file to build protobuf files in $scriptDirectory = $targetFolder\script + # the script file can be used to create the compiled protobuf libraries in $targetPath = $targetFolder\$prodSubDir + + $prodName = "ProtoBuf 3.1.0 Source" + $prodFile = "protobuf310.zip" + $prodName = "protobuf-3.1.0" + $prodSubDir = "protobuf-3.1.0-vs15" + $batchFile = "buildProto.cmd" + + $protoSourceDir = join-path $targetFolder "src" + $scriptDirectory = join-path $targetFolder "script" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "PROTOBUF_PATH" + $envValue = $targetPath + $downloadSource = "https://github.com/google/protobuf/archive/v3.1.0.zip" + + @( @{ShortName = "PROTO310VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $protoSourceDir; zipSubTree =$prodName; destinationFolder =$prodName }, + @{Function = "MakeDirectory"; Path = $scriptDirectory }, + @{Function = "CreateBuildProtobufBatch"; FileName = "$scriptDirectory\$batchFile"; SourceDir = (join-path $protoSourceDir $prodName); TargetDir = $targetPath } ); + } ) +} + +function OpProtoBuf310VS15Internal( + [parameter(Mandatory=$true)][string] $server, + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "ProtoBuf 3.1.0 Prebuild VS2015" + $prodFile = "PreBuildProtobuf310vs15.zip" + $prodSubDir = "protobuf-3.1.0-vs15" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "PROTOBUF_PATH" + $envValue = $targetPath + $downloadSource = "$server\$prodFile" + + @( @{ShortName = "PROTO310VS15PRE"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree ="protobuf-3.1.0-vs15"; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + } ) +} + +function OpScanProgram() +{ + @{ ShortName = "SCANPROG"; VerifyInfo = "Scan System for installed programs"; + Verification = @( @{Function = "VerifyScanPrograms" } ) + } +} + +function OpSwig3010( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "SWIG 3.0.10" + $prodFile = "swigwin-3.0.10.zip" + $prodSubDir = "swigwin-3.0.10" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "SWIG_PATH" + $envValue = $targetPath + $downloadSource = "http://prdownloads.sourceforge.net/project/swig/swigwin/swigwin-3.0.10/swigwin-3.0.10.zip" + + @( @{ShortName = "SWIG3010"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + } ) +} + +function OpSysinternals( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Sysinternal Suite" + $prodFile = "SysinternalSuite.zip" + $prodDependsFile = "depends22_x64.zip" + $prodSubDir = "SysInternal" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://download.sysinternals.com/files/SysinternalsSuite.zip" + $downloadDependsSource = "http://dependencywalker.com/depends22_x64.zip" + + + @( @{ShortName = "SYSINTERNAL"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName, dependency walker"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyFile"; Path = "$targetPath\depends.exe" }, + @{Function = "VerifyPathIncludes"; Path = $targetPath } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" }, + @{ Function = "Download"; Source = $downloadDependsSource; Destination = "$cache\$prodDependsFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; destinationFolder =$prodSubDir }, + @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodDependsFile"; destination = $targetFolder; destinationFolder =$prodSubDir; AddToDirectory=$true }, + @{Function = "AddToPath"; Dir = $targetPath; AtStart = $false; } + @{Function = "SetRegistryKey"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals" }, + @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\Autologon"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" }, + @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\Handle"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" }, + @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\ProcDump"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" } ) + } ) +} + +function OpTestData( + [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $remoteData) +{ + $prodName = "Testdata Environment" + @( @{ShortName = "TESTDATA"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Setting up environment variable for $prodName"; + Verification = @( @{Function = "VerifyEnvironment"; EnvVar = "CNTK_EXTERNAL_TESTDATA_REMOTE_DIRECTORY" }, + @{Function = "VerifyEnvironment"; EnvVar = "CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY" } ); + Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = "CNTK_EXTERNAL_TESTDATA_REMOTE_DIRECTORY"; Content = $remoteData }, + @{Function = "SetEnvironmentVariable"; EnvVar = "CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY"; Content = $targetFolder } ); + } ) +} + +function OpAddVS12Runtime([parameter(Mandatory=$true)][string] $cache) +{ + $prodName = "VS2012 Runtime" + + @( @{ShortName = "VS2012"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2012 x64 Additional Runtime" }, + @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2012 x64 Minimum Runtime" } ); + Download = @( @{ Function = "Download"; Source = "http://download.microsoft.com/download/1/6/B/16B06F60-3B20-4FF2-B699-5E9B7962F9AE/VSU_4/vcredist_x64.exe"; Destination = "$cache\VSRuntime\11\vcredist_x64.exe" } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\VSRuntime\11\vcredist_x64.exe"; Param = "/install /passive /norestart" } ) + } ) +} + +function OpAddVS13Runtime([parameter(Mandatory=$true)][string] $cache) +{ + $prodName = "VS2013 Runtime" + + @( @{ShortName = "VS2013"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2013 x64 Additional Runtime" }, + @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2013 x64 Minimum Runtime" } ); + Download = @( @{ Function = "Download"; Source = "http://download.microsoft.com/download/2/E/6/2E61CFA4-993B-4DD4-91DA-3737CD5CD6E3/vcredist_x64.exe"; Destination = "$localCache\VSRuntime\12\vcredist_x64.exe" } ); + Action = @( @{Function = "InstallExe"; Command = "$localCache\VSRuntime\12\vcredist_x64.exe"; Param = "/install /passive /norestart" } ) + } ) +} + +function OpCheckVS15Update3 +{ + @( @{Name = "Verify Installation of VS2015, Update 3"; ShortName = "PREVS15U3"; VerifyInfo = "Checking for Visual Studio 2015, Update 3"; + Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft Build Tools 14.0 \(amd64\)$"; Version = "14.0.25420"; MatchExact = $true} ); + PreReq = @( @{Function = "PrereqInfoVS15" } ); + Action = @( @{Function = "StopInstallation" } ) + } ) +} + +function OpCheckCuda8 +{ + $programPath = join-path $env:ProgramFiles "NVIDIA GPU Computing Toolkit\CUDA\v8.0" + @( @{Name = "Verify Installation of NVidia Cuda 8"; ShortName = "PRECUDA8"; VerifyInfo = "Checking for NVidia Cuda 8"; + Verification = @( @{Function = "VerifyDirectory"; Path = $programPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "CUDA_PATH_V8_0"; Content = $programPath } ); + PreReq = @( @{Function = "PrereqInfoCuda8" } ); + Action = @( @{Function = "StopInstallation" } ) + } ) +} + +function OpZlib( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "ZLib128" + $prodFile = "zlib128.zip" + $prodSubDir = "zlib-1.2.8" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "http://zlib.net/zlib128.zip" + + @( @{ShortName = "ZLIN128"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir } ); + } ) +} + +function OpLibzip( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Libzip-1.1.3" + $prodFile = "libzip-1.1.3.tar.gz" + $prodSubDir = "libzip-1.1.3" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://nih.at/libzip/libzip-1.1.3.tar.gz" + + @( @{ShortName = "LIBZIP113"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir } ); + } ) +} + + +function OpZLibInternal( + [parameter(Mandatory=$true)][string] $server, + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "ZLib Precompiled" + $prodFile = "zlib.zip" + $prodSubDir = "zlib" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "ZLIB_PATH" + $envValue = $targetPath + $downloadSource = "$server\$prodFile" + + @( @{ShortName = "LIBZIP113"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; destinationFolder =$prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + } ) +} + +function OpZlibVS15( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + # unzip protobuf source in $protoSourceDir = $targetfolder\src\$prodsubdir + # create batch file to build protobuf files in $scriptDirectory = $targetFolder\script + # the script file can be used to create the compiled protobuf libraries in $targetPath = $targetFolder\$prodSubDir + + $prodName = "zlib / libzip from source" + $zlibProdName = "zlib-1.2.8" + $zlibFilename = "zlib128.zip" + $zlibDownloadSource = "http://zlib.net/zlib128.zip" + $libzipProdName = "libzip-1.1.3" + $libzipFilename = "libzip-1.1.3.tar.gz" + $libzipDownloadSource = "https://nih.at/libzip/libzip-1.1.3.tar.gz" + + $prodSubDir = "zlib-vs15" + $batchFile = "buildZlibVS15.cmd" + + $sourceCodeDir = join-path $targetFolder "src" + $scriptDirectory = join-path $targetFolder "script" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "ZLIB_PATH" + $envValue = $targetPath + + @( @{ShortName = "ZLIBVS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = "$sourceCodeDir\$zlibProdName" }, + @{Function = "VerifyDirectory"; Path = "$sourceCodeDir\$libzipProdName" }, + @{Function = "VerifyFile"; Path = "$scriptDirectory\$batchFile" } ); + Download = @( @{ Function = "Download"; Source = $zlibDownloadSource; Destination = "$cache\$zlibFilename" }, + @{ Function = "Download"; Source = $libzipDownloadSource; Destination = "$cache\$libzipFilename" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$zlibFilename"; destination = $sourceCodeDir; zipSubTree =$zlibProdName; destinationFolder =$zlibProdName }, + @{Function = "ExtractAllFromTarGz"; SourceFile = "$cache\$libzipFilename"; TargzFileName = "$libzipFilename"; destination = $sourceCodeDir }, + @{Function = "MakeDirectory"; Path = $scriptDirectory }, + @{Function = "CreateBuildZlibBatch"; FileName = "$scriptDirectory\$batchFile"; zlibSourceDir = (join-path $sourceCodeDir $zlibProdName); libzipSourceDir = (join-path $sourceCodeDir $libzipProdName); TargetDir = $targetPath } ); + } ) +} + + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/PreRequisites.ps1 b/Scripts/devInstall/Windows/helper/PreRequisites.ps1 new file mode 100644 index 000000000000..259f48081904 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/PreRequisites.ps1 @@ -0,0 +1,95 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function PreReqOperations( + [array] $actionList = @()) +{ + foreach ($item in $actionList) { + foreach ($prereqItem in $item.PreReq) { + PreRequisiteItem $prereqItem + } + } + + Write-Host "Install operations finished" + Write-Host +} + +function PreRequisiteItem( + [hashtable] $item) +{ + $func = $item["Function"] + + $expr = $func +' $item' + + Write-Verbose "Calling Operation: [$func]" + Invoke-Expression $expr +} + +function PrereqInfo2013Up5( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + + Write-Host " + +We require the installation of Visual Studio 2013 Update 5 to continue. +Please follow the information here [https://support.microsoft.com/en-us/kb/3021976]. +" +} + +function PrereqInfoVS15( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + Write-Host " + +Installation of Visual Studio 2015 Update 3 is a pre-requisite before installation +can continue. Please check [https://www.visualstudio.com/vs/] for more details on +Visual Studion 2015 +" +return +} + +function PrereqInfoCuda8( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + Write-Host " + +Installation of NVidia Cuda 8.0 is a pre-requisite before installation +can continue. Please check https://developer.nvidia.com/cuda-downloads] for more +details on Cuda download. Please also remember to set the environment variable +CUDA_PATH_V8_0 to the Cuda8 installation location. +" +return +} + +function PrereqInfoCuDnn51( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + Write-Host " + +Installation of NVidia CuDNN 4.1 for Cuda 8.0 is a pre-requisite before installation +can continue. Please check https://developer.nvidia.com/cudn] for more +details on Cuda download. Please also remember to set the environment variable +CUDNN_PATH to the CudDnn folder in your installation location. +" +return +} + +function StopInstallation( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + throw "Not all pre-requisites installed, installation terminated." +} + + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Verification.ps1 b/Scripts/devInstall/Windows/helper/Verification.ps1 new file mode 100644 index 000000000000..c69905f95ac4 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Verification.ps1 @@ -0,0 +1,366 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function VerifyOperations( + [array] $verificationList) +{ + Write-Host "Determining Operations to perform. This will take a moment..." + + $result = @() + + foreach ($item in $verificationList) { + $needsInstall = $false + Write-Host $item.VerifyInfo + foreach ($verificationItem in $item.Verification) { + $needsInstall = VerifyItem $verificationItem + if (-not $needsInstall) { + $result += $item + break + } + } + } + return $result +} + +function VerifyItem( + [hashtable] $item) +{ + $func = $item["Function"] + $name = $item["Name"] + + $expr = $func +' $item' + + Write-Verbose "Calling Operation: [$func]: [$name]" + $noInstallRequired = Invoke-Expression $expr + return $noInstallRequired +} + +function VerifyScanPrograms( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $noInstallRequired = $true + + # no actual work is being performed, just the script local datastructure with the list + # of installed programs is being initialized + LoadWinProduct + return $noInstallRequired +} + +function VerifyWinProductExists( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $match = $table["Match"] + $compare = GetTableDefaultString -table $table -entryName "Compare" -defaultValue "" + $matchExact = GetTableDefaultBool -table $table -entryName "MatchExact" -defaultValue $true + $noInstallRequired = $true + + $allProducts = LoadWinProduct + $productList = @($allProducts | Where-Object { $_.Name -match $match } ) + + if ($productList.Count -eq 0) { + $noInstallRequired = $false + } + elseif ($compare.length -gt 0) { + if ($matchExact) { + $productList = @($productList | Where-Object { $_.Name -eq $compare }) + } + else { + $productList = @($productList | Where-Object { $_.Version -ge $compare }) + } + if ($productList.Count -eq 0) { + Write-Verbose "No product found matching the compare requirement`n$productList" + $noInstallRequired = $false + } + } + + Write-Verbose "[$func]: Product [$match] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyWinProductVersion( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $match = $table["Match"] + $version = $table["Version"] + $matchExact = GetTableDefaultBool -table $table -entryName "MatchExact" -defaultValue $false + $noInstallRequired = $true + + if ($matchExact -eq $null) { + $matchExact = $false + } + + $allProducts = LoadWinProduct + $productList = @($allProducts | Where-Object { $_.Name -match $match } ) + + if ($productList.Count -eq 0) { + Write-Verbose "No product found with Name matching [$match]" + $noInstallRequired = $false + } + else { + if ($matchExact) { + $productList = @($productList | Where-Object { $_.Version -eq $version }) + } + else { + $productList = @($productList | Where-Object { $_.Version -ge $version }) + } + if ($productList.Count -eq 0) { + Write-Verbose "No product found matching the version requirement`n$productList" + $noInstallRequired = $false + } + } + Write-Verbose "[$func]: Product [$match] Version {$version] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyInstallationContent( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["Path"] + + $noInstallRequired = (join-path $path cntk\cntk.exe | test-path -PathType Leaf) + $noInstallRequired = (join-path $path prerequisites\VS2012\vcredist_x64.exe | test-path -PathType Leaf) -and $noInstallRequired + $noInstallRequired = (join-path $path prerequisites\VS2013\vcredist_x64.exe | test-path -PathType Leaf) -and $noInstallRequired + $noInstallRequired = (join-path $path prerequisites\MSMpiSetup.exe | test-path -PathType Leaf) -and $noInstallRequired + + if ($noInstallRequired) { + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired + } + + throw "`nFatal Error: Files from CNTK binary download package are missing!`nThe install script must be run out of the unpacked binary CNTK package, not from a CNTK source clone." +} + +function VerifyDirectory( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["Path"] + + $noInstallRequired = (test-path -path $path -PathType Container) + + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyRunAlways( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + + $noInstallRequired = $false + Write-Verbose "[$func]: returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyWheelDirectory( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["WheelDirectory"] + $forceUpdate = $table["ForceUpdate"] + + $noInstallRequired = $false + + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyPathIncludes( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["Path"] + + $noInstallRequired = (test-path -path $path -PathType Container) + + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyDirectoryContent( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $source = $table["Source"] + $dest = $table["Destination"] + + $noInstallRequired = (test-path -path $source -PathType Container) + + if ($noInstallRequired) { + $noInstallRequired = (test-path -path $dest -PathType Container) + } + if ($noInstallRequired) { + $r = Compare-Object $(Get-ChildItem $source -Recurse) $(Get-ChildItem $dest -Recurse) + if ($r) { + $noInstallRequired = $false + } + } + + Write-Verbose "[$func]: [$source] with [$dest] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyFile( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["Path"] + + $noInstallRequired = (test-path -path $path -PathType Leaf) + + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyRegistryKey( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["Key"] + + $noInstallRequired = (test-path -path $key) + + Write-Verbose "[$func]: [$key] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyRegistryKeyName( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["Key"] + $regName = $table["RegName"] + + $noInstallRequired = Test-ItemProperty -Path $key -Name $regName + + Write-Verbose "[$func]: [$key]:[$regname] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyRegistryKeyNameData( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["Key"] + $regName = $table["RegName"] + $regData = $table["RegData"] + $orLater = $table["OrLater"] + + if ($orLater -eq $null) { + $orLater = $false + } + + $noInstallRequired = (test-path -path $key) + if ($noInstallRequired) { + $theKeyObj = get-item $key + $theKeyValue = $theKeyObj.GetValue("$regName") + $noInstallRequired = $false + + if ($theKeyValue -ne $null) { + if ($orLater) { + $noInstallRequired = ($theKeyValue -ge $regData) + } + else { + $noInstallRequired = ($theKeyValue -eq $regData) + } + } + } + + Write-Verbose "[$func]: [$key]:[$regname] == [$regData] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyEnvironmentAndData( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $name = $table["EnvVar"] + $content = $table["Content"] + $location = "Machine" + + $envContent = [environment]::GetEnvironmentVariable($name, $location) + + $noInstallRequired = ($envContent -ieq $content) + + Write-Verbose "[$func]: [$name] == [$content] returned [$noInstallRequired]" + return $noInstallRequired +} + +function Test-ItemProperty ( + [string] $Path, + [string] $Name) +{ + if (Test-Path $Path) { + try { + $ItemProperty = Get-ItemProperty -Path $Path -Name $Name -ErrorAction SilentlyContinue + if ( $ItemProperty -ne $null ) { + return $true + } + } + catch { + return $false + } + } + return $false +} + +function ClearScriptVariables +{ + $Script:WinProduct = $Null +} + +function LoadWinProduct +{ + if ($Script:WinProduct -eq $Null) { + # + # $Script:WinProduct = Get-WmiObject Win32_Product + # The above line was the previous solution, but iterating through the registry is much faster + # get-wmiobject does more house-holding, like checking for concistency etc ... + # + $allInstalled = @(Get-childitem "HKCU:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall" -ErrorAction SilentlyContinue) + ` + @(Get-childitem "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall" -ErrorAction SilentlyContinue) + ` + @(get-childitem "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\" -ErrorAction SilentlyContinue) + + $result = @() + foreach($item in $allInstalled) { + $displayName = $item.GetValue("DisplayName") + if ($displayName.Length -gt 0) { + $entry = New-Object PSObject + $entry | Add-Member -MemberType NoteProperty -Name "Name" -Value $displayName + $entry | Add-Member -MemberType NoteProperty -Name "Version" -Value $($item.GetValue("DisplayVersion")) + + $result += $entry + } + + } + $result = $result | Sort-Object Name,Version -Unique + + $Script:WinProduct = $result + } + return $Script:WinProduct +} + + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/readme.md b/Scripts/devInstall/Windows/readme.md new file mode 100644 index 000000000000..38d2062bc556 --- /dev/null +++ b/Scripts/devInstall/Windows/readme.md @@ -0,0 +1,118 @@ +## Developer Setup Windows from a script + +This documents describes the setup of the CNTK developer environment. The script follows overall the recipe described in the public documentation of the CNTK developer setup, available [here](https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows). + +>Note: Before startingt the script base installation, please check the CNTK Wiki for more detailed instructions (?here?) + +### The installation script + +The most up-to-date version of the installation script can be found on ``\\cntk-muc00\devinstall``. Copy +The prepared installation script to a directory on your system. In the following we assume you are +going to copy the files in to the directory `c:\local\devinstall`. Start a Windows command shell and enter +``` +c: +cd \ +md local && cd local +md devinstall && cd devinstall +robocopy \\cntk-muc00\devinstall . /MIR +``` +The installation script itself is written for the Powershell environment. By default Powershell doesn't +allow the execution of any scripts. To allow execution of the installation script, +start Powershell from a **standard Windows command shell** by: +``` +start powershell -executionpolicy remotesigned +``` +This will open a Powershell environment. In this enviroment change into the directory you copied +the install script into, here you can directly start the installation script +``` +cd c:\local\devinstall +.\devinstall.ps1 +``` +>Note: If you now receive an errormessage stating that running scripts is disabled on your system, +please make sure you started powershell with the changed executionpolicy. This is enabling +script execution in your created Powershell environment. + +The script will inspect your system and determine the components which are missing to build the +Microsoft Cognitive Toolkit. You will be +notified about the proposed installation steps. At this point you are running in a **demo** mode - +NO changes to your system are being performed. + +### Running the installation script + +If you are satisfied with the proposed changes, you can proceed to the actual installation. +The installation script supports several command line options. ``get-help .\install.ps1`` will +give you a list over the available option. At this point it is recommended to start the installation +by adding the `-Execute` parameter: +``` +.\devinstall.ps1 -execute +``` +The script will download needed components from the web, therefore a connection to the Internet is required. +Downloaded components are stored in the directory ``c:\cntkLocalCache`` and can be removed after complete +installation. During execution of the installation script you might receive requests/warnings from UAC +(User Account Control) depending on your system configuration. Please acknowledge the execution and +installation of the downloaded components. + +> Note: Some components of the installation script (i.e. NVidia CUDA install ) might performan a reboot of your +system. You can just start the installation script again. It will analyze your system again and reuse already +downloaded components. + +### Result of the installation script + +The following changes (if necessary) to your system will be performed by the installation script: + +- Installation of the Visual Studio 2012 and Visual Studio 2013 runtime modules +- Installation of Microsoft MPI +- Installation of the Microsoft MPI SDK +- If not present on your system. Git will be installed + - Location: ``%PROGAMFILES%\Git`` + - Environment: Added to the PATH +- NVidia CUDA 7.5 + - Location: ``%PROGRAMFILES(x86)%\NVIDIA Corporation``, ``%PROGRAMFILES%\NVIDIA GPU Computing Toolkit``, + see NVidia documentation for details + - Environment: ``CUDA_PATH``, ``CUDA_PATH_V7_5`` +- NVidia CuDNN 5.1 for CUDA 7.5 + - Location: ``c:\local\cudnn-7.5-v5.1`` + - Environment: ``CUDNN_PATH`` +- NVidia CUB 1.4.1 + - Location: ``c:\local\cub-1.4.1`` + - Environment: ``CUB_PATH`` +- Boost 1.60 + - Location: ``c:\local\boost_1_60_0`` + - Environment: ``BOOST_INCLUDE_PATH``, ``BOOST_LIB_PATH`` +- A CNTK specific MKL library + - Location: ``c:\local\cntkmkl`` + - Environment: ``CNTK_MKL_PATH`` +- OpenCV 3.1.0 + - Location: ``c:\local\Opencv3.1.0`` + - Environment: ``OPENCV_PATH_V31`` +- Zlib - Precompiled version of Zlib and libzip + - Location: ``c:\local\zlib`` + - Environment: ``ZLIB_PATH`` +- Protobuf 3.1.0 libararies and headers + - Location: ``c:\local\protobuf-3.1.0`` + - Environment: ``PROTOBUF_PATH`` +- SWIG 3.0.10 + - Location: ``c:\local\swigwin-3.0.10`` + - Environment: ``SWIG_PATH`` +- CNTK Git repository clone + - Location: ``c:\repos\CNTK`` +- Anaconda3 - 4.1.1 + - Location: ``c:\local\Anaconda3-4.1.1-Windows-x86_64`` + This is a local Anaconda install, it hasn't been added to the path, and is registered only to the current user + +>Note: This script already installed the compiled Protobuf library, as well as the compiled +zlib and libzip components. It isn't necessary to perform the additional compilation steps (listed in the +public documentation) for these components. + +Once the script finished, you should be ready to build. A couple of points to note: + - Depending on the tools installed, the script might have done changes to your system (especially a CUDA install). + Rebooting you system might be a good choice. + - It is possible to run the script multiple times. Powershell is very specific that it doesn't pick up changes to + environment variables which are done in the script, you have to restart powershell to pick up the latest environment. + - If you are planning on using a GPU, you should install the latest GPU driver from NVidia, and reboot + your system. + + + + + From 0923497dccf27070ec4ea6f6572c6453a9041136 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Thu, 5 Jan 2017 19:29:10 +0100 Subject: [PATCH 076/120] fixed boost download and opencv installation --- Scripts/devInstall/Windows/DevInstall.ps1 | 13 +- Scripts/devInstall/Windows/helper/Action.ps1 | 60 ++- .../devInstall/Windows/helper/Download.ps1 | 8 +- .../devInstall/Windows/helper/Operations.ps1 | 372 ++---------------- 4 files changed, 87 insertions(+), 366 deletions(-) diff --git a/Scripts/devInstall/Windows/DevInstall.ps1 b/Scripts/devInstall/Windows/DevInstall.ps1 index ffa06f536704..a0459d64d46e 100644 --- a/Scripts/devInstall/Windows/DevInstall.ps1 +++ b/Scripts/devInstall/Windows/DevInstall.ps1 @@ -57,11 +57,6 @@ Param( [parameter(Mandatory=$false)] [string] $ServerLocation, [parameter(Mandatory=$false)] [string] $CloneDirectory) - - - $Execute = $true - - $roboCopyCmd = "robocopy.exe" $localDir = $InstallLocation @@ -141,6 +136,7 @@ Function main $operation += OpSwig3010 -cache $localCache -targetFolder $localDir $operation += OpProtoBuf310VS15 -cache $localCache -targetFolder $localDir $operation += OpZlibVS15 -cache $localCache -targetFolder $localDir + $operation += OpOpenCV31 -cache $localCache -targetFolder $localDir if ($ServerLocation) { $operation += OpProtoBuf310VS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir $operation += OpZLibVS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir @@ -153,12 +149,7 @@ Function main #$operation += OpGitClone -targetFolder $repositoryRootDir -targetDir $reponame #$operation += OpSysinternals -cache $localCache -targetFolder $localDir #$operation += OpOpenCVInternal $ServerLocation -cache $localCache -targetFolder $localDir - #$operation += OpOpenCV31 -cache $localCache -targetFolder $localDir - #$operation += OpCygwin -cache $localCache -targetFolder $localDir - - #$operation += AddOpDisableJITDebug - #$operation += OpTestData "c:\Data\CNTKTestData" "\\storage.ccp.philly.selfhost.corp.microsoft.com\public\CNTKTestData" - #$operation += OpSysinternals -cache $localCache -targetFolder $localDir + $operationList = @() diff --git a/Scripts/devInstall/Windows/helper/Action.ps1 b/Scripts/devInstall/Windows/helper/Action.ps1 index d8f8c8bce29f..00abf8cbcdd3 100644 --- a/Scripts/devInstall/Windows/helper/Action.ps1 +++ b/Scripts/devInstall/Windows/helper/Action.ps1 @@ -326,30 +326,66 @@ function ExtractAllFromZip( } $tempDir = [System.IO.Path]::GetTempFileName(); - remove-item $tempDir | Out-Null $completeTempDestination = join-path -Path $tempDir -ChildPath $destinationFolder new-item -type directory -path $completeTempDestination -Force -ErrorAction Stop | Out-Null + $obj = new-object -com shell.application + $zipFile = $obj.NameSpace($zipFileName) + $destinationNS = $obj.NameSpace($completeTempDestination) - if ($Execute) { - $obj = new-object -com shell.application - $zipFile = $obj.NameSpace($zipFileName) - $destinationNS = $obj.NameSpace($completeTempDestination) + $destinationNS.CopyHere($zipFile.Items()) - $destinationNS.CopyHere($zipFile.Items()) + if ($zipSubTree) { + $completeTempDestination = join-path $completeTempDestination $zipSubTree + } - if ($zipSubTree -ne $null) { - $completeTempDestination = join-path $completeTempDestination $zipSubTree - } + RobocopySourceDestination $completeTempDestination $completeDestination $copyAdditive + + rm -r $tempDir -Force -ErrorAction SilentlyContinue | Out-Null +} + + +function Extract7zipSelfExtractingArchive( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $archiveName = $table["archiveName"] + $destination = $table["destination"] + $destinationFolder = $table["destinationFolder"] + $archiveSubTree = $table["archiveSubTree"] + $copyAdditive = GetTableDefaultBool -table $table -entryName "AddToDirectory" -defaultValue $false + + Write-Verbose "Extract7zipSelfExtractingArchive: archiveName [$archiveName] destination [$destination] Folder [$destinationFolder]" - RobocopySourceDestination $completeTempDestination $completeDestination $copyAdditive + if (-not $Execute) { + return + } + $completeDestination = join-path -Path $destination -ChildPath $destinationFolder - rm -r tempDir -Force -ErrorAction SilentlyContinue | Out-Null + if (-not (test-path $archiveName -PathType Leaf)) { + throw "Extract7zipSelfExtractingArchive: zipFileName [$zipFileName] not found!" } - return + $tempDir = [System.IO.Path]::GetTempFileName(); + remove-item $tempDir | Out-Null + + $completeTempDestination = $tempDir + new-item -type directory -path $completeTempDestination -Force -ErrorAction Stop | Out-Null + + $cmdParm = "-o`"$completeTempDestination`" -y" + $newTable = @{ Function = "InstallExe"; Command = $archiveName; Param = $cmdParm; runAs=$false } + InstallExe $newTable + + if ($archiveSubTree) { + $completeTempDestination = join-path $completeTempDestination $archiveSubTree + } + RobocopySourceDestination $completeTempDestination $completeDestination $copyAdditive + + rm -r $tempDir -Force -ErrorAction SilentlyContinue | Out-Null } function ExtractAllFromTarGz( diff --git a/Scripts/devInstall/Windows/helper/Download.ps1 b/Scripts/devInstall/Windows/helper/Download.ps1 index 6bc4c6771187..56ae8fe8b3eb 100644 --- a/Scripts/devInstall/Windows/helper/Download.ps1 +++ b/Scripts/devInstall/Windows/helper/Download.ps1 @@ -61,7 +61,7 @@ function Download( DownloadFileWebRequest -SourceFile $source -OutFile $destination -ExpectedSize $ExpectedSize } else { - DownloadFile -SourceFile $source -OutFile $destination + DownloadFileWebClient -SourceFile $source -OutFile $destination } return } @@ -130,7 +130,7 @@ function DownloadAndExtract( ){ $outFileName = Join-Path $tPath $fileName - DownloadFile -SourceFile $sAddress ` + DownloadFileWebClient -SourceFile $sAddress ` -OutFile $outFileName ` -tempFileName $fileName @@ -189,7 +189,7 @@ function DownloadFileWebRequest ( Remove-Item -path $TempFile -ErrorAction SilentlyContinue } -function DownloadFile( +function DownloadFileWebClient( [string] $SourceFile, [string] $OutFile, [int] $timeout = 600, @@ -203,7 +203,7 @@ function DownloadFile( #--------------- $startTime = Get-Date - Write-Host "Downloading [$SourceFile], please be patient...." + Write-Host "Downloading [$SourceFile], please be patient, no progress message is shown ..." if (-not $Execute) { Write-Host "$message ** Running in DEMOMODE - no download performed" return diff --git a/Scripts/devInstall/Windows/helper/Operations.ps1 b/Scripts/devInstall/Windows/helper/Operations.ps1 index f2980af245db..7e1d87515707 100644 --- a/Scripts/devInstall/Windows/helper/Operations.ps1 +++ b/Scripts/devInstall/Windows/helper/Operations.ps1 @@ -40,27 +40,6 @@ function OpAnacondaEnv34( } } -function OpBoost160( - [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) -{ - $prodName = "Boost 1.60.0" - $prodFile = "boost_1_60_0-msvc-12.0-64.exe" - $prodSubDir = "boost_1_60_0" - $targetPath = join-path $targetFolder $prodSubDir - $downloadSource = "https://sourceforge.net/projects/boost/files/boost-binaries/1.60.0/boost_1_60_0-msvc-12.0-64.exe/download" - - @( @{Name = $prodName; ShortName = "BOOST160"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, - @{Function = "VerifyEnvironmentAndData"; EnvVar = "BOOST_INCLUDE_PATH"; Content = $targetPath }, - @{Function = "VerifyEnvironmentAndData"; EnvVar = "BOOST_LIB_PATH"; Content = "$targetPath\lib64-msvc-12.0" } ); - Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = "BOOST_INCLUDE_PATH"; Content = "$targetPath" }, - @{Function = "SetEnvironmentVariable"; EnvVar = "BOOST_LIB_PATH"; Content = "$targetPath\lib64-msvc-12.0" }, - @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=$targetPath /SP- /SILENT /NORESTART"; runAs=$false } ); - } ) -} - function OpBoost160VS15( [parameter(Mandatory=$true)][string] $cache, [parameter(Mandatory=$true)][string] $targetFolder) @@ -73,13 +52,14 @@ function OpBoost160VS15( $envVar = "BOOST_INCLUDE_PATH" $envVarLib = "BOOST_LIB_PATH" $envContentLib = "$targetPath\lib64-msvc-14.0" + $downloadSize = 0 @( @{Name = $prodName; ShortName = "BOOST160VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $targetPath }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVarLib; Content = $envContentLib } ); - Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $targetPath }, + Download = @( @{Function = "Download"; Method = "WebClient"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); + Actionssssss = @( @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $targetPath }, @{Function = "SetEnvironmentVariable"; EnvVar = $envVarLib; Content = $envContentLib }, @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=$targetPath /SP- /SILENT /NORESTART"; runAs=$false } ); } ) @@ -92,37 +72,16 @@ function OpCMake362( $targetPath = join-path $env:ProgramFiles "cmake\bin" $cmakeName = "cmake-3.6.2-win64-x64.msi" $downloadSource = "https://cmake.org/files/v3.6/cmake-3.6.2-win64-x64.msi" + $downloadSize = 15771063 @( @{ShortName = "CMake362"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyWinProductExists"; Match = "^CMake$"; Version = "3.6.2" } ); - Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$cmakeName" } ); + Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$cmakeName"; ExpectedSize = $downloadSize } ); Action = @( @{Function = "InstallMsi"; MsiName = "$cmakeName" ; MsiDir = "$cache" } , @{Function = "AddToPath"; Dir = "$targetPath" } ); } ) } -function OpCNTKMKL2 - ([parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) -{ - $prodName = "CNTK Custom MKL Version 2" - $prodFile = "CNTKCustomMKL-Windows-2.zip" - $prodSubDir = "CNTKCustomMKL" - $targetPath = join-path $targetFolder $prodSubDir - $targetPathCurrenVersion = join-path $targetPath "2" - $envVar = "CNTK_MKL_PATH"; - $envValue = $targetPath - $downloadSource = "https://www.cntk.ai/mkl/$prodFile"; - - @( @{ShortName = "CNTKMKL2"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPathCurrenVersion"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPathCurrenVersion }, - @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $localDir; destinationFolder = $prodSubDir }, - @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); - } ) -} - function OpCNTKMKL3 ([parameter(Mandatory=$true)][string] $cache, [parameter(Mandatory=$true)][string] $targetFolder) @@ -135,100 +94,28 @@ function OpCNTKMKL3 $envVar = "CNTK_MKL_PATH"; $envValue = $targetPath $downloadSource = "https://www.cntk.ai/mkl/$prodFile"; + $downloadSize = 10331909 @( @{ShortName = "CNTKMKL3"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPathCurrenVersion"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPathCurrenVersion }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $localDir; destinationFolder = $prodSubDir }, + Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; destinationFolder = $prodSubDir }, @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); } ) } -function OpCygwin - ([parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) -{ - $prodName = "CygWin64" - $prodFile = "cygwinSetup_64.exe" - $prodSubDir = "cygwin64" - $targetPath = join-path $targetFolder $prodSubDir - $regkey = "Registry::HKEY_CURRENT_USER\SOFTWARE\Cygwin" - $downloadSource = "https://www.cygwin.com/setup-x86_64.exe" - $installParam = "--quiet-mode --no-admin --site http://mirrors.kernel.org/sourceware/cygwin/ --root $targetPath --local-package-dir $cache --packages python,python-yaml,python-numpy,diffutils" - $bFileName = "cygwinpip.bash" - $bashFileName = join-path $targetFolder $bFileName - $bashParmFile = Join-Path "\cygdrive\c" (split-path $bashFileName -NoQualifier) - $bashParmFile = $bashParmFile.replace("\","/") - $bashParam = "-l -c $bashParmFile" - - @( @{ShortName = "CYGWIN64"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, - @{Function = "VerifyRegistryKey"; Key = $regKey } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = $installParam; runAs=$false; WorkDir = $targetFolder}, - @{Function = "AddToPath"; Dir = "$targetPath\bin"; AtStart = $false }, - @{Function = "CreateCygwinBashScript"; FileName = $bashFileName }, - @{Function = "InstallExe"; Command = "$targetPath\bin\bash.exe"; Param = $bashParam; runAs=$false; WorkDir = $targetPath } ); - } ) -} - -function AddOpDisableJITDebug -{ - @( @{Name="Visual Studio Disable Just-In-Time Debugging"; ShortName = "JITVS2013"; VerifyInfo = "Checking for JIT Debugging registry keys"; ActionInfo = "Removing registry keys to disable JIT debugging, advisable for Jenkins machines"; - Verification = @( @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug"; RegName = "Debugger"; }, - @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework"; RegName = "DbgManagedDebugger" }, - @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows NT\CurrentVersion\AeDebug"; RegName = "Debugger" }, - @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\.NETFramework"; RegName = "DbgManagedDebugger" } ); - Action = @( @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug"; Elevated = $true; RegName = "Debugger" }, - @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework"; Elevated = $true; RegName = "DbgManagedDebugger" }, - @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows NT\CurrentVersion\AeDebug"; Elevated = $true; RegName = "Debugger" }, - @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\.NETFramework"; Elevated = $true; RegName = "DbgManagedDebugger" } ) - } ) - } - -function OpGit2101( - [parameter(Mandatory=$true)][string] $cache) -{ - $prodName = "Git" - $targetPath = join-path $env:ProgramFiles "Git\bin" - $prodFile = "Git-2.10.1-64-bit.exe" - $downloadSource = "https://github.com/git-for-windows/git/releases/download/v2.10.1.windows.1/$prodFile" - - @( @{ShortName = "GIT"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\GitForWindows"; RegName = "CurrentVersion"; } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/SP- /SILENT /NORESTART"}, - @{Function = "AddToPath"; Dir = $targetPath; AtStart = $true; } ) - } ) -} - -function OpGitClone( - [parameter(Mandatory=$true)][string] $targetFolder, - [parameter(Mandatory=$true)][string] $targetDir, - [string] $repoTag = "master") -{ - $targetPath = join-path $targetFolder $targetDir - $downloadSource = "https://github.com/Microsoft/CNTK/"; - $appDir = join-path $env:ProgramFiles "Git\bin" - - @( @{Name = "Clone CNTK from Github"; ShortName = "CNTKCLONE"; VerifyInfo = "Checking for CNTK-Clone target directory $targetPath"; ActionInfo = "Cloneing CNTK from Github repository"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); - Action = @( @{Function = "MakeDirectory"; Path = $targetFolder }, - @{Function = "ExecuteApplication"; AppName = "git.exe"; Param = "clone --branch $repoTag --recursive https://github.com/Microsoft/CNTK/"; AppDir = $appDir; UseEnvPath = $true; WorkDir = $targetFolder } ) - } ) -} - function OpMSMPI70([parameter( Mandatory=$true)][string] $cache) { $remoteFilename = "MSMpiSetup.exe" $localFilename = "MSMpiSetup70.exe" $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; + $downloadSize = 2285568 @( @{Name = "MSMPI Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70"; ActionInfo = "Installing MSMPI 70"; Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); - Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$localFilename" } ); + Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$localFilename"; ExpectedSize = $downloadSize } ); Action = @( @{Function = "InstallExe"; Command = "$cache\$localFilename" ; Param = "/unattend" } ) } ) } @@ -239,11 +126,12 @@ function OpMSMPI70SDK( $remoteFilename = "msmpisdk.msi" $localFilename = "msmpisdk70.msi" $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; + $downloadSize = 5277808 @( @{Name = "MSMPI SDK70 Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70 SDK"; ActionInfo = "Install MSMPI 70 SDK"; Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI SDK \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); #Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft MPI SDK \(\d+\."; Compare = "^Microsoft MPI SDK \(7\.0\.12437\.6\)"; MatchExact = $false } ); - Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$localFilename" } ); + Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$localFilename"; ExpectedSize = $downloadSize } ); Action = @( @{Function = "InstallMsi"; MsiName = "$localFilename" ; MsiDir = "$cache" } ) } ) } @@ -259,78 +147,17 @@ function OpNvidiaCub141( $envVar = "CUB_PATH"; $envValue = $targetPath $downloadSource = "https://codeload.github.com/NVlabs/cub/zip/1.4.1"; + $downloadSize = 51376046 @( @{ShortName = "CUB141"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = "$targetFolder"; destinationFolder = $prodSubDir; zipSubTree= $prodSubDir }, @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); } ) } -function OpNVidiaCuda75( - [parameter(Mandatory=$true)][string] $cache) -{ - $cudaDownload = "http://developer.download.nvidia.com/compute/cuda/7.5/Prod/local_installers" - $programPath = join-path $env:ProgramFiles "NVIDIA GPU Computing Toolkit\CUDA\v7.5" - - - @( @{Name="NVidia CUDA 7.5"; ShortName = "CUDA75"; VerifyInfo = "Checking for installed NVidia Cuda 75"; ActionInfo = "Installing CUDA 7.5"; - Verification = @( @{Function = "VerifyDirectory"; Path = $programPath }, - @{Function = "VerifyEnvironmentAndData"; EnvVar = "CUDA_PATH_V7_5"; Content = $programPath } ); - Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Source = "$cudaDownload/cuda_7.5.18_windows.exe"; Destination = "$cache\cuda_7.5.18_windows.exe" }, - @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 10"; Source = "$cudaDownload/cuda_7.5.18_win10.exe"; Destination = "$cache\cuda_7.5.18_win10.exe" } ); - Action = @( @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Command = "$cache\cuda_7.5.18_windows.exe"; Param = "-s CUDAToolkit_7.5 CUDAVisualStudioIntegration_7.5 GDK"; Message = ".... This will take some time. Please be patient ...." } , - @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows 10"; Command = "$cache\cuda_7.5.18_win10.exe"; Param = "-s CUDAToolkit_7.5 CUDAVisualStudioIntegration_7.5 GDK"; Message = ".... This will take some time. Please be patient ...." } ); - }) -} - -function OpNVidiaCuda80( - [parameter(Mandatory=$true)][string] $cache) -{ - $cudaDownload = "http://developer.download.nvidia.com/compute/cuda/8.0/secure/prod/local_installers" - $cudaFile = "cuda_8.0.44_windows.exe" - $cudaFileWin10 = "cuda_8.0.44_win10.exe" - $programPath = join-path $env:ProgramFiles "NVIDIA GPU Computing Toolkit\CUDA\v8.0" - $cudaInstallParam = "-s" - - @( @{Name="NVidia CUDA 8.0"; ShortName = "CUDA80"; VerifyInfo = "Checking for installed NVidia Cuda 80"; ActionInfo = "Installing CUDA 8.0"; - Verification = @( @{Function = "VerifyDirectory"; Path = $programPath }, - @{Function = "VerifyEnvironmentAndData"; EnvVar = "CUDA_PATH_V8_0"; Content = $programPath } ); - Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Source = "$cudaDownload/$cudaFile"; Destination = "$cache\$cudaFile" }, - @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 10"; Source = "$cudaDownload/$cudaFileWin10"; Destination = "$cache\$cudaFileWin10" } ); - Action = @( @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Command = "$cache\$cudaFile"; Param = $cudaInstallParam } , - @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows 10"; Command = "$cache\$cudaFileWin10"; Param = $cudaInstallParam } ); - }) -} - -function OpNVidiaCudnn5175( - [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) - -{ - $prodName = "NVidia CUDNN 5.1 for CUDA 7.5" - $cudnnWin7 = "cudnn-7.5-windows7-x64-v5.1.zip" - $cudnnWin10 = "cudnn-7.5-windows10-x64-v5.1.zip" - - $prodSubDir = "cudnn-7.5-v5.1" - $targetPath = join-path $targetFolder $prodSubDir - $envVar = "CUDNN_PATH" - $envValue = join-path $targetPath "cuda" - $downloadSource = "http://developer.download.nvidia.com/compute/redist/cudnn/v5.1" - - @( @{ShortName = "CUDNN5175"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, - @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 7"; Source = "$downloadSource/$cudnnWin7"; Destination = "$cache\$cudnnWin7" }, - @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; Source = "$downloadSource/$cudnnWin10"; Destination = "$cache\$cudnnWin10" } ); - Action = @( @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows 7"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, - @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, - @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); - }) -} - function OpNVidiaCudnn5180( [parameter(Mandatory=$true)][string] $cache, [parameter(Mandatory=$true)][string] $targetFolder) @@ -344,13 +171,15 @@ function OpNVidiaCudnn5180( $envVar = "CUDNN_PATH" $envValue = join-path $targetPath "cuda" $downloadSource = "http://developer.download.nvidia.com/compute/redist/cudnn/v5.1" + $downloadSizeWin7 = 0 + $downloadSizeWin10 = 54440432 @( @{ShortName = "CUDNN5180"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, @{Function = "VerifyDirectory"; Path = $envValue }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{Function = "DownloadForPlatform"; Platform = "^Microsoft Windows 7"; Source = "$downloadSource/$cudnnWin7"; Destination = "$cache\$cudnnWin7" }, - @{Function = "DownloadForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; Source = "$downloadSource/$cudnnWin10"; Destination = "$cache\$cudnnWin10" } ); + Download = @( @{Function = "DownloadForPlatform"; Method = "WebRequest"; Platform = "^Microsoft Windows 7"; Source = "$downloadSource/$cudnnWin7"; Destination = "$cache\$cudnnWin7"; ExpectedSize = $downloadSizeWin7 }, + @{Function = "DownloadForPlatform"; Method = "WebRequest"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; Source = "$downloadSource/$cudnnWin10"; Destination = "$cache\$cudnnWin10"; ExpectedSize = $downloadSizeWin10 } ); Action = @( @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows 7"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); @@ -363,36 +192,19 @@ function OpOpenCV31( { $prodName = "OpenCV-3.1" $prodFile = "opencv-3.1.0.exe" - $prodSubDir = "OpenCV310" - $targetPath = join-path $targetFolder $prodSubDir - $envVar = "OPENCV_PATH_V31"; - $envValue = "$targetPath\build" - $downloadSource = "https://netcologne.dl.sourceforge.net/project/opencvlibrary/opencv-win/3.1.0/opencv-3.1.0.exe" - - @( @{ShortName = "OPENCV310"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Please perform a manual installation of $prodName from $cache"; - Verification = @( @{Function = "VerifyFile"; Path = "$cache\$prodFile" } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - } ) -} - -function OpOpenCVInternal( - [parameter(Mandatory=$true)][string] $server, - [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) -{ - $prodName = "OpenCV-3.1" - $prodFile = "opencv-3.1.0.zip" $prodSubDir = "Opencv3.1.0" $targetPath = join-path $targetFolder $prodSubDir $envVar = "OPENCV_PATH_V31"; $envValue = "$targetPath\build" - $downloadSource = "$server\$prodFile" + $downloadSource = "https://netcologne.dl.sourceforge.net/project/opencvlibrary/opencv-win/3.1.0/opencv-3.1.0.exe" + $downloadSize = 115129966 + $archiveSubTree = "opencv" @( @{ShortName = "OPENCV310"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{ Function = "LocalCopyFile"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = "$targetFolder"; destinationFolder = $prodSubDir; zipSubTree= $prodSubDir }, + Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); + Action = @( @{Function = "Extract7zipSelfExtractingArchive"; archiveName = "$cache\$prodFile"; destination = "$targetFolder"; destinationFolder = $prodSubDir; archiveSubTree= $archiveSubTree }, @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); } ) } @@ -417,10 +229,11 @@ function OpProtoBuf310VS15( $envVar = "PROTOBUF_PATH" $envValue = $targetPath $downloadSource = "https://github.com/google/protobuf/archive/v3.1.0.zip" + $downloadSize = 5648581 @( @{ShortName = "PROTO310VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize} ); Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $protoSourceDir; zipSubTree =$prodName; destinationFolder =$prodName }, @{Function = "MakeDirectory"; Path = $scriptDirectory }, @{Function = "CreateBuildProtobufBatch"; FileName = "$scriptDirectory\$batchFile"; SourceDir = (join-path $protoSourceDir $prodName); TargetDir = $targetPath } ); @@ -467,82 +280,17 @@ function OpSwig3010( $envVar = "SWIG_PATH" $envValue = $targetPath $downloadSource = "http://prdownloads.sourceforge.net/project/swig/swigwin/swigwin-3.0.10/swigwin-3.0.10.zip" + $downloadSize = 56429 @( @{ShortName = "SWIG3010"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir }, @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); } ) } -function OpSysinternals( - [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) -{ - $prodName = "Sysinternal Suite" - $prodFile = "SysinternalSuite.zip" - $prodDependsFile = "depends22_x64.zip" - $prodSubDir = "SysInternal" - $targetPath = join-path $targetFolder $prodSubDir - $downloadSource = "https://download.sysinternals.com/files/SysinternalsSuite.zip" - $downloadDependsSource = "http://dependencywalker.com/depends22_x64.zip" - - - @( @{ShortName = "SYSINTERNAL"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName, dependency walker"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, - @{Function = "VerifyFile"; Path = "$targetPath\depends.exe" }, - @{Function = "VerifyPathIncludes"; Path = $targetPath } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" }, - @{ Function = "Download"; Source = $downloadDependsSource; Destination = "$cache\$prodDependsFile" } ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; destinationFolder =$prodSubDir }, - @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodDependsFile"; destination = $targetFolder; destinationFolder =$prodSubDir; AddToDirectory=$true }, - @{Function = "AddToPath"; Dir = $targetPath; AtStart = $false; } - @{Function = "SetRegistryKey"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals" }, - @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\Autologon"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" }, - @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\Handle"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" }, - @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\ProcDump"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" } ) - } ) -} - -function OpTestData( - [parameter(Mandatory=$true)][string] $targetFolder, - [parameter(Mandatory=$true)][string] $remoteData) -{ - $prodName = "Testdata Environment" - @( @{ShortName = "TESTDATA"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Setting up environment variable for $prodName"; - Verification = @( @{Function = "VerifyEnvironment"; EnvVar = "CNTK_EXTERNAL_TESTDATA_REMOTE_DIRECTORY" }, - @{Function = "VerifyEnvironment"; EnvVar = "CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY" } ); - Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = "CNTK_EXTERNAL_TESTDATA_REMOTE_DIRECTORY"; Content = $remoteData }, - @{Function = "SetEnvironmentVariable"; EnvVar = "CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY"; Content = $targetFolder } ); - } ) -} - -function OpAddVS12Runtime([parameter(Mandatory=$true)][string] $cache) -{ - $prodName = "VS2012 Runtime" - - @( @{ShortName = "VS2012"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2012 x64 Additional Runtime" }, - @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2012 x64 Minimum Runtime" } ); - Download = @( @{ Function = "Download"; Source = "http://download.microsoft.com/download/1/6/B/16B06F60-3B20-4FF2-B699-5E9B7962F9AE/VSU_4/vcredist_x64.exe"; Destination = "$cache\VSRuntime\11\vcredist_x64.exe" } ); - Action = @( @{Function = "InstallExe"; Command = "$cache\VSRuntime\11\vcredist_x64.exe"; Param = "/install /passive /norestart" } ) - } ) -} - -function OpAddVS13Runtime([parameter(Mandatory=$true)][string] $cache) -{ - $prodName = "VS2013 Runtime" - - @( @{ShortName = "VS2013"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2013 x64 Additional Runtime" }, - @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2013 x64 Minimum Runtime" } ); - Download = @( @{ Function = "Download"; Source = "http://download.microsoft.com/download/2/E/6/2E61CFA4-993B-4DD4-91DA-3737CD5CD6E3/vcredist_x64.exe"; Destination = "$localCache\VSRuntime\12\vcredist_x64.exe" } ); - Action = @( @{Function = "InstallExe"; Command = "$localCache\VSRuntime\12\vcredist_x64.exe"; Param = "/install /passive /norestart" } ) - } ) -} - function OpCheckVS15Update3 { @( @{Name = "Verify Installation of VS2015, Update 3"; ShortName = "PREVS15U3"; VerifyInfo = "Checking for Visual Studio 2015, Update 3"; @@ -563,63 +311,6 @@ function OpCheckCuda8 } ) } -function OpZlib( - [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) -{ - $prodName = "ZLib128" - $prodFile = "zlib128.zip" - $prodSubDir = "zlib-1.2.8" - $targetPath = join-path $targetFolder $prodSubDir - $downloadSource = "http://zlib.net/zlib128.zip" - - @( @{ShortName = "ZLIN128"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir } ); - } ) -} - -function OpLibzip( - [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) -{ - $prodName = "Libzip-1.1.3" - $prodFile = "libzip-1.1.3.tar.gz" - $prodSubDir = "libzip-1.1.3" - $targetPath = join-path $targetFolder $prodSubDir - $downloadSource = "https://nih.at/libzip/libzip-1.1.3.tar.gz" - - @( @{ShortName = "LIBZIP113"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir } ); - } ) -} - - -function OpZLibInternal( - [parameter(Mandatory=$true)][string] $server, - [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) -{ - $prodName = "ZLib Precompiled" - $prodFile = "zlib.zip" - $prodSubDir = "zlib" - $targetPath = join-path $targetFolder $prodSubDir - $envVar = "ZLIB_PATH" - $envValue = $targetPath - $downloadSource = "$server\$prodFile" - - @( @{ShortName = "LIBZIP113"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, - @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; destinationFolder =$prodSubDir }, - @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); - } ) -} - function OpZlibVS15( [parameter(Mandatory=$true)][string] $cache, [parameter(Mandatory=$true)][string] $targetFolder) @@ -632,9 +323,12 @@ function OpZlibVS15( $zlibProdName = "zlib-1.2.8" $zlibFilename = "zlib128.zip" $zlibDownloadSource = "http://zlib.net/zlib128.zip" + $downloadSizeZlib = 0 + $libzipProdName = "libzip-1.1.3" $libzipFilename = "libzip-1.1.3.tar.gz" $libzipDownloadSource = "https://nih.at/libzip/libzip-1.1.3.tar.gz" + $downloadeSizeLibzip = 0 $prodSubDir = "zlib-vs15" $batchFile = "buildZlibVS15.cmd" @@ -649,8 +343,8 @@ function OpZlibVS15( Verification = @( @{Function = "VerifyDirectory"; Path = "$sourceCodeDir\$zlibProdName" }, @{Function = "VerifyDirectory"; Path = "$sourceCodeDir\$libzipProdName" }, @{Function = "VerifyFile"; Path = "$scriptDirectory\$batchFile" } ); - Download = @( @{ Function = "Download"; Source = $zlibDownloadSource; Destination = "$cache\$zlibFilename" }, - @{ Function = "Download"; Source = $libzipDownloadSource; Destination = "$cache\$libzipFilename" } ); + Download = @( @{ Function = "Download"; Source = $zlibDownloadSource; Destination = "$cache\$zlibFilename"; ExpectedSize = $downloadSizeZlib }, + @{ Function = "Download"; Source = $libzipDownloadSource; Destination = "$cache\$libzipFilename"; ExpectedSize = $downloadeSizeLibzip } ); Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$zlibFilename"; destination = $sourceCodeDir; zipSubTree =$zlibProdName; destinationFolder =$zlibProdName }, @{Function = "ExtractAllFromTarGz"; SourceFile = "$cache\$libzipFilename"; TargzFileName = "$libzipFilename"; destination = $sourceCodeDir }, @{Function = "MakeDirectory"; Path = $scriptDirectory }, From eac7334254f0ff7028cac52b85ff744446ccfcf9 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Sun, 8 Jan 2017 13:41:58 +0000 Subject: [PATCH 077/120] several fixes for downloading, boost and protobuf install --- .../devInstall/Windows/helper/Operations.ps1 | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Scripts/devInstall/Windows/helper/Operations.ps1 b/Scripts/devInstall/Windows/helper/Operations.ps1 index 7e1d87515707..a856055d6d44 100644 --- a/Scripts/devInstall/Windows/helper/Operations.ps1 +++ b/Scripts/devInstall/Windows/helper/Operations.ps1 @@ -59,7 +59,7 @@ function OpBoost160VS15( @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $targetPath }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVarLib; Content = $envContentLib } ); Download = @( @{Function = "Download"; Method = "WebClient"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); - Actionssssss = @( @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $targetPath }, + Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $targetPath }, @{Function = "SetEnvironmentVariable"; EnvVar = $envVarLib; Content = $envContentLib }, @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=$targetPath /SP- /SILENT /NORESTART"; runAs=$false } ); } ) @@ -111,7 +111,7 @@ function OpMSMPI70([parameter( $remoteFilename = "MSMpiSetup.exe" $localFilename = "MSMpiSetup70.exe" $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; - $downloadSize = 2285568 + $downloadSize = 5277808 @( @{Name = "MSMPI Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70"; ActionInfo = "Installing MSMPI 70"; Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); @@ -126,7 +126,7 @@ function OpMSMPI70SDK( $remoteFilename = "msmpisdk.msi" $localFilename = "msmpisdk70.msi" $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; - $downloadSize = 5277808 + $downloadSize = 2285568 @( @{Name = "MSMPI SDK70 Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70 SDK"; ActionInfo = "Install MSMPI 70 SDK"; Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI SDK \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); @@ -197,13 +197,13 @@ function OpOpenCV31( $envVar = "OPENCV_PATH_V31"; $envValue = "$targetPath\build" $downloadSource = "https://netcologne.dl.sourceforge.net/project/opencvlibrary/opencv-win/3.1.0/opencv-3.1.0.exe" - $downloadSize = 115129966 + $downloadSize = 0 $archiveSubTree = "opencv" @( @{ShortName = "OPENCV310"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); + Download = @( @{ Function = "Download"; Method = "WebClient"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); Action = @( @{Function = "Extract7zipSelfExtractingArchive"; archiveName = "$cache\$prodFile"; destination = "$targetFolder"; destinationFolder = $prodSubDir; archiveSubTree= $archiveSubTree }, @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); } ) @@ -232,7 +232,7 @@ function OpProtoBuf310VS15( $downloadSize = 5648581 @( @{ShortName = "PROTO310VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Verification = @( @{Function = "VerifyDirectory"; Path = $protoSourceDir } ); Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize} ); Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $protoSourceDir; zipSubTree =$prodName; destinationFolder =$prodName }, @{Function = "MakeDirectory"; Path = $scriptDirectory }, @@ -280,12 +280,12 @@ function OpSwig3010( $envVar = "SWIG_PATH" $envValue = $targetPath $downloadSource = "http://prdownloads.sourceforge.net/project/swig/swigwin/swigwin-3.0.10/swigwin-3.0.10.zip" - $downloadSize = 56429 + $downloadSize = 0 @( @{ShortName = "SWIG3010"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); + Download = @( @{ Function = "Download"; Method = "WebClient"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir }, @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); } ) @@ -320,9 +320,9 @@ function OpZlibVS15( # the script file can be used to create the compiled protobuf libraries in $targetPath = $targetFolder\$prodSubDir $prodName = "zlib / libzip from source" - $zlibProdName = "zlib-1.2.8" - $zlibFilename = "zlib128.zip" - $zlibDownloadSource = "http://zlib.net/zlib128.zip" + $zlibProdName = "zlib-1.2.10" + $zlibFilename = "zlib1210.zip" + $zlibDownloadSource = "http://zlib.net/zlib1210.zip" $downloadSizeZlib = 0 $libzipProdName = "libzip-1.1.3" From 4ac873f91ae35574f991233677f8b95c96e7198c Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Mon, 9 Jan 2017 17:44:35 +0100 Subject: [PATCH 078/120] fix zlib download, create parameterized batch files for protobuf and zlib compilation --- Scripts/devInstall/Windows/DevInstall.ps1 | 4 +- Scripts/devInstall/Windows/buildProtoVS15.bat | 108 ++++++++++++++++ Scripts/devInstall/Windows/buildZlibVS15.bat | 108 ++++++++++++++++ Scripts/devInstall/Windows/helper/Action.ps1 | 116 +++--------------- .../devInstall/Windows/helper/Operations.ps1 | 23 ++-- 5 files changed, 245 insertions(+), 114 deletions(-) create mode 100644 Scripts/devInstall/Windows/buildProtoVS15.bat create mode 100644 Scripts/devInstall/Windows/buildZlibVS15.bat diff --git a/Scripts/devInstall/Windows/DevInstall.ps1 b/Scripts/devInstall/Windows/DevInstall.ps1 index a0459d64d46e..b9f43894492f 100644 --- a/Scripts/devInstall/Windows/DevInstall.ps1 +++ b/Scripts/devInstall/Windows/DevInstall.ps1 @@ -134,8 +134,8 @@ Function main $operation += OpBoost160VS15 -cache $localCache -targetFolder $localDir $operation += OpCNTKMKL3 -cache $localCache -targetFolder $localDir $operation += OpSwig3010 -cache $localCache -targetFolder $localDir - $operation += OpProtoBuf310VS15 -cache $localCache -targetFolder $localDir - $operation += OpZlibVS15 -cache $localCache -targetFolder $localDir + $operation += OpProtoBuf310VS15 -cache $localCache -targetFolder $localDir -repoDirectory $CloneDirectory + $operation += OpZlibVS15 -cache $localCache -targetFolder $localDir -repoDirectory $CloneDirectory $operation += OpOpenCV31 -cache $localCache -targetFolder $localDir if ($ServerLocation) { $operation += OpProtoBuf310VS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir diff --git a/Scripts/devInstall/Windows/buildProtoVS15.bat b/Scripts/devInstall/Windows/buildProtoVS15.bat new file mode 100644 index 000000000000..ac560e51983e --- /dev/null +++ b/Scripts/devInstall/Windows/buildProtoVS15.bat @@ -0,0 +1,108 @@ +@REM Copyright (c) Microsoft. All rights reserved. +@REM Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +@REM +@REM batch script to build protobuf library for CNTK + +@echo off +if /I "%CMDCMDLINE%" neq ""%COMSPEC%" " ( + @echo. + @echo Please execute this script from inside a regular Windows command prompt. + @echo. + exit /b 0 +) + +if "%1"=="" ( goto HELP ) +if "%1"=="-?" ( goto HELP ) +if /I "%1"=="-h" ( goto HELP ) +if /I "%1"=="-help" ( goto HELP ) +if "%2"=="" ( goto HELP ) +if not "%3"=="" ( goto HELP ) + +SET SOURCEDIR=%~f1 +SET TARGETDIR=%~f2 + +IF "%SOURCEDIR:~-1%"=="\" SET SOURCEDIR=%SOURCEDIR:~,-1% +IF "%TARGETDIR:~-1%"=="\" SET TARGETDIR=%TARGETDIR:~,-1% + +if not exist "%SOURCEDIR%\build" ( + @echo Error: "%SOURCEDIR%" is not a valid ProtoBuf source directory + goto FIN +) + +set cmakePath= +for %%f in ("cmake.exe") do set cmakePath=%%~$PATH:f +if "%cmakePath%"=="" ( + @echo Error: CMAKE.EXE not found in PATH! + goto FIN +) + +SET VCDIRECTORY=%VS140COMNTOOLS% +if "%VCDIRECTORY%"=="" ( + @echo Environment variable VS140COMNTOOLS not defined. + @echo Make sure Visual Studion 2015 Update 3 is installed. + goto FIN +) + +IF "%VCDIRECTORY:~-1%"=="\" SET VCDIRECTORY=%VCDIRECTORY:~,-1% + +if not exist "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" ( + echo Error: "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" not found. + echo Make sure you have installed Visual Studion 2015 Update 3 correctly. + goto FIN +) + +@echo. +@echo This will build Protobuf-3.1.0 using Visual Studio 2015 +@echo ------------------------------------------------------- +@echo The configured settings for the batch file: +@echo Visual Studio directory: %VCDIRECTORY% +@echo Protobuf source directory: %SOURCEDIR% +@echo Protobuf target directory: %TARGETDIR% +@echo. + +pause + +call "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" amd64 + +pushd "%SOURCEDIR%" +cd cmake +md build && cd build + +md debug && cd debug +cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" ..\.. +nmake +nmake install +cd .. + +md release && cd release +cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" ..\.. +nmake +nmake install +cd .. + +popd + +setx PROTOBUF_PATH %TARGETDIR% +set PROTOBUF_PATH=%TARGETDIR% + +goto FIN + + + +:HELP +@echo. +@echo Use this script to build the Protobuf library for CNTK. +@echo The script requires two parameter +@echo Parameter 1: The complete path to the ProtoBuf source directory +@echo i.e: C:\local\src\protobuf-3.1.0 +@echo Parameter 2: The target path for the created binaries +@echo i.e. C:\local\protobuf-3.1.0-vs15 +@echo The sript will also set the environment variable PROTOBUF_PATH to +@echo the target directory +@echo. +goto FIN + +:FIN + + +REM vim:set expandtab shiftwidth=2 tabstop=2: diff --git a/Scripts/devInstall/Windows/buildZlibVS15.bat b/Scripts/devInstall/Windows/buildZlibVS15.bat new file mode 100644 index 000000000000..3f85cc8117a6 --- /dev/null +++ b/Scripts/devInstall/Windows/buildZlibVS15.bat @@ -0,0 +1,108 @@ +@REM Copyright (c) Microsoft. All rights reserved. +@REM Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +@REM +@REM batch script to build image compression libraries for CNTK + +@echo off +if /I "%CMDCMDLINE%" neq ""%COMSPEC%" " ( + @echo. + @echo Please execute this script from inside a regular Windows command prompt. + @echo. + exit /b 0 +) + +if "%1"=="" ( goto HELP ) +if "%1"=="-?" ( goto HELP ) +if /I "%1"=="-h" ( goto HELP ) +if /I "%1"=="-help" ( goto HELP ) +if "%2"=="" ( goto HELP ) +if "%3"=="" ( goto HELP ) +if not "%4"=="" ( goto HELP ) + +SET LIBZIPSOURCEDIR=%~f1 +set ZLIBSOURCEDIR=%~f2 +SET TARGETDIR=%~f3 + +IF "%LIBZIPSOURCEDIR:~-1%"=="\" SET LIBZIPSOURCEDIR=%LIBZIPSOURCEDIR:~,-1% +IF "%ZLIBSOURCEDIR:~-1%"=="\" SET ZLIBSOURCEDIR=%ZLIBSOURCEDIR:~,-1% +IF "%TARGETDIR:~-1%"=="\" SET TARGETDIR=%TARGETDIR:~,-1% + +if not exist "%LIBZIPSOURCEDIR%\CMakeLists.txt" ( + @echo Error: "%LIBZIPSOURCEDIR%" not a valid LibZib directory + goto :FIN +) +if not exist "%ZLIBSOURCEDIR%\CMakeLists.txt" ( + @echo Error: "%ZLIBSOURCEDIR%" not a valid ZLib directory + goto :FIN +) + +SET VCDIRECTORY=%VS140COMNTOOLS% +if "%VCDIRECTORY%"=="" ( + @echo Environment variable VS140COMNTOOLS not defined. + @echo Make sure Visual Studion 2015 Update 3 is installed. + goto FIN +) + +IF "%VCDIRECTORY:~-1%"=="\" SET VCDIRECTORY=%VCDIRECTORY:~,-1% + +if not exist "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" ( + echo Error: "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" not found. + echo Make sure you have installed Visual Studion 2015 Update 3 correctly. + goto FIN +) + +@SET CMAKEGEN="Visual Studio 14 2015 Win64" + +@echo. +@echo This will build compression libraries for CNTK using Visual Studio 2015 +@echo ----------------------------------------------------------------------- +@echo The configured settings for the batch file: +@echo Visual Studio directory: %VCDIRECTORY% +@echo CMake Generator: %CMAKEGEN% +@echo LibZip source directory: %LIBZIPSOURCEDIR% +@echo Zlib source directory: %ZLIBSOURCEDIR% +@echo Zlib-VS15 target directory: %TARGETDIR% +@echo. + +@pause + +call "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" amd64 + +@pushd "%ZLIBSOURCEDIR%" +@mkdir build +@cd build +@cmake .. -G%CMAKEGEN% -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" +@msbuild /P:Configuration=Release INSTALL.vcxproj +@popd + +@pushd "%LIBZIPSOURCEDIR%" +@md build +@cd build +@cmake .. -G%CMAKEGEN% -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" +@msbuild libzip.sln /t:zip /P:Configuration=Release +@cmake -DBUILD_TYPE=Release -P cmake_install.cmake +@popd + +setx ZLIB_PATH %TARGETDIR% +set ZLIB_PATH=%TARGETDIR% + +goto FIN + +:HELP +@echo. +@echo Use this script to build the image compression libraries for CNTK. +@echo The script requires three parameter +@echo Parameter 1: The complete path to the LibZip source directory +@echo i.e: C:\local\src\libzip-1.1.3 +@echo Parameter 1: The complete path to the ZLib source directory +@echo i.e: C:\local\src\zlib-1.2.8 +@echo Parameter 2: The target path for the created binaries +@echo i.e. C:\local\zlib-vs15 +@echo The sript will also set the environment variable ZLIB_PATH to +@echo the target directory +@echo. +goto FIN + +:FIN + +REM vim:set expandtab shiftwidth=2 tabstop=2: diff --git a/Scripts/devInstall/Windows/helper/Action.ps1 b/Scripts/devInstall/Windows/helper/Action.ps1 index 00abf8cbcdd3..8d7d351a2c85 100644 --- a/Scripts/devInstall/Windows/helper/Action.ps1 +++ b/Scripts/devInstall/Windows/helper/Action.ps1 @@ -523,11 +523,12 @@ function CreateBuildProtobufBatch( $filename = $table["FileName"] $sourceDir = $table["SourceDir"] $targetDir = $table["TargetDir"] + $repoDirectory = $table["repoDirectory"] if ($Execute) { Remove-Item -Path $filename -ErrorAction SilentlyContinue | Out-Null - $batchScript = GetBatchBuildProtoBuf $sourceDir $targetDir + $batchScript = GetBatchBuildProtoBuf $sourceDir $targetDir $repoDirectory add-content -Path $filename -Encoding Ascii -Value $batchScript } @@ -543,11 +544,12 @@ function CreateBuildZlibBatch( $zlibSourceDir = $table["zlibSourceDir"] $libzipSourceDir = $table["libzipSourceDir"] $targetDir = $table["TargetDir"] + $repoDirectory = $table["repoDirectory"] if ($Execute) { Remove-Item -Path $filename -ErrorAction SilentlyContinue | Out-Null - $batchScript = GetBatchBuildZlibBuf $zlibSourceDir $libzipSourceDir $targetDir + $batchScript = GetBatchBuildZlibBuf $zlibSourceDir $libzipSourceDir $targetDir $repoDirectory add-content -Path $filename -Encoding Ascii -Value $batchScript } @@ -703,116 +705,26 @@ function CallGetCommand( function GetBatchBuildProtoBuf( [string] $sourceDir, - [string] $targetDir) + [string] $targetDir, + [string] $repoDirectory) { - $batchscript = @" -@SET VCDIRECTORY=C:\Program Files (x86)\Microsoft Visual Studio 14.0 -@SET SOURCEDIR=$sourceDir -@SET TARGETDIR=$targetDir - -@echo. -@echo This will build Protobuf-3.1.0 -@echo ------------------------------ -@echo The configured settings for the batch file: -@echo Visual Studio directory: %VCDIRECTORY% -@echo Protobuf source directory: %SOURCEDIR% -@echo Protobuf target directory: %TARGETDIR% -@echo. -@echo. -@echo Please edit the batch file if this doesn't match your directory layout! -@echo. - -@pause - -@call "%VCDIRECTORY%\VC\vcvarsall.bat" amd64 - -@pushd %SOURCEDIR% -@cd cmake -@md build && cd build - -@md debug && cd debug -@cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" ..\.. -@nmake -@nmake install -@cd .. - -@md release && cd release -@cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" ..\.. -@nmake -@nmake install -@cd .. - -@popd - -setx PROTOBUF_PATH %TARGETDIR% + $batchFile = join-path $repoDirectory "Scripts\devInstall\Windows\buildProtoVS15.bat" +@" +call $batchFile $sourceDir $targetDir "@ - - return $batchscript } function GetBatchBuildZlibBuf( [string] $zlibSourceDir, [string] $libzipSourceDir, - [string] $targetDir) + [string] $targetDir, + [string] $repoDirectory) { - $batchscript = @" -@SET VCDIRECTORY=C:\Program Files (x86)\Microsoft Visual Studio 14.0 -@SET LIBZIPSOURCEDIR=$libzipSourceDir -@SET ZLIBSOURCEDIR=$zlibSourceDir -@SET TARGETDIR=$targetDir -@SET CMAKEGEN="Visual Studio 14 2015 Win64" - -@echo. -@echo This will build ZLib using Visual Studio 2015 -@echo --------------------------------------------- -@echo The configured settings for the batch file: -@echo Visual Studio directory: %VCDIRECTORY% -@echo CMake Generator: %CMAKEGEN% -@echo LibZip source directory: %LIBZIPSOURCEDIR% -@echo Zlib source directory: %ZLIBSOURCEDIR% -@echo Zlib-VS15 target directory: %TARGETDIR% -@echo. -@echo. -@echo Please edit the batch file if this doesn't match your directory layout! -@echo. - -@pause - -@call "%VCDIRECTORY%\VC\vcvarsall.bat" amd64 - -@pushd %ZLIBSOURCEDIR% -@mkdir build -@cd build -@cmake .. -G%CMAKEGEN% -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" -@msbuild /P:Configuration=Release INSTALL.vcxproj -@popd - -@pushd %LIBZIPSOURCEDIR% -@md build -@cd build -@cmake .. -G%CMAKEGEN% -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" -@msbuild libzip.sln /t:zip /P:Configuration=Release -@cmake -DBUILD_TYPE=Release -P cmake_install.cmake -@popd - -setx ZLIB_PATH %TARGETDIR% -"@ - - return $batchscript -} - + $batchFile = join-path $repoDirectory "Scripts\devInstall\Windows\buildZlibVS15.bat" - -function GetCygwinBashScript -{ - $batchscript = @" -easy_install-2.7 pip -pip install six -pip install pytest +@" +call $batchFile $libzipSourceDir $zlibSourceDir $targetDir "@ - - return $batchscript } - # vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Operations.ps1 b/Scripts/devInstall/Windows/helper/Operations.ps1 index a856055d6d44..fb4fac7f7c37 100644 --- a/Scripts/devInstall/Windows/helper/Operations.ps1 +++ b/Scripts/devInstall/Windows/helper/Operations.ps1 @@ -211,7 +211,8 @@ function OpOpenCV31( function OpProtoBuf310VS15( [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) + [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $repoDirectory) { # unzip protobuf source in $protoSourceDir = $targetfolder\src\$prodsubdir # create batch file to build protobuf files in $scriptDirectory = $targetFolder\script @@ -221,9 +222,10 @@ function OpProtoBuf310VS15( $prodFile = "protobuf310.zip" $prodName = "protobuf-3.1.0" $prodSubDir = "protobuf-3.1.0-vs15" - $batchFile = "buildProto.cmd" + $batchFile = "buildProtoVS15.cmd" $protoSourceDir = join-path $targetFolder "src" + $completeProtoSourceDir = join-Path $protoSourceDir $prodName $scriptDirectory = join-path $targetFolder "script" $targetPath = join-path $targetFolder $prodSubDir $envVar = "PROTOBUF_PATH" @@ -232,11 +234,11 @@ function OpProtoBuf310VS15( $downloadSize = 5648581 @( @{ShortName = "PROTO310VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $protoSourceDir } ); + Verification = @( @{Function = "VerifyDirectory"; Path = $completeProtoSourceDir } ); Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize} ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $protoSourceDir; zipSubTree =$prodName; destinationFolder =$prodName }, + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $protoSourceDir; zipSubTree = $prodName; destinationFolder = $prodName }, @{Function = "MakeDirectory"; Path = $scriptDirectory }, - @{Function = "CreateBuildProtobufBatch"; FileName = "$scriptDirectory\$batchFile"; SourceDir = (join-path $protoSourceDir $prodName); TargetDir = $targetPath } ); + @{Function = "CreateBuildProtobufBatch"; FileName = "$scriptDirectory\$batchFile"; SourceDir = $completeProtoSourceDir; TargetDir = $targetPath; RepoDirectory = $repoDirectory } ); } ) } @@ -313,16 +315,17 @@ function OpCheckCuda8 function OpZlibVS15( [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) + [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $repoDirectory) { # unzip protobuf source in $protoSourceDir = $targetfolder\src\$prodsubdir # create batch file to build protobuf files in $scriptDirectory = $targetFolder\script # the script file can be used to create the compiled protobuf libraries in $targetPath = $targetFolder\$prodSubDir $prodName = "zlib / libzip from source" - $zlibProdName = "zlib-1.2.10" - $zlibFilename = "zlib1210.zip" - $zlibDownloadSource = "http://zlib.net/zlib1210.zip" + $zlibProdName = "zlib-1.2.8" + $zlibFilename = "zlib128.zip" + $zlibDownloadSource= "https://netix.dl.sourceforge.net/project/libpng/zlib/1.2.8/zlib128.zip" $downloadSizeZlib = 0 $libzipProdName = "libzip-1.1.3" @@ -348,7 +351,7 @@ function OpZlibVS15( Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$zlibFilename"; destination = $sourceCodeDir; zipSubTree =$zlibProdName; destinationFolder =$zlibProdName }, @{Function = "ExtractAllFromTarGz"; SourceFile = "$cache\$libzipFilename"; TargzFileName = "$libzipFilename"; destination = $sourceCodeDir }, @{Function = "MakeDirectory"; Path = $scriptDirectory }, - @{Function = "CreateBuildZlibBatch"; FileName = "$scriptDirectory\$batchFile"; zlibSourceDir = (join-path $sourceCodeDir $zlibProdName); libzipSourceDir = (join-path $sourceCodeDir $libzipProdName); TargetDir = $targetPath } ); + @{Function = "CreateBuildZlibBatch"; FileName = "$scriptDirectory\$batchFile"; zlibSourceDir = (join-path $sourceCodeDir $zlibProdName); libzipSourceDir = (join-path $sourceCodeDir $libzipProdName); TargetDir = $targetPath; RepoDirectory = $repoDirectory } ); } ) } From 5d28532c4b90ff3113ac33a0114d33de969905a0 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Tue, 10 Jan 2017 09:36:47 +0100 Subject: [PATCH 079/120] fix VerifyEnvironmentAndData, integrate download and install of prebuild protobuf and zlib --- Scripts/devInstall/Windows/DevInstall.ps1 | 14 +---- Scripts/devInstall/Windows/helper/Action.ps1 | 15 +---- .../devInstall/Windows/helper/Operations.ps1 | 60 +++++++++++++------ .../Windows/helper/Verification.ps1 | 7 ++- 4 files changed, 49 insertions(+), 47 deletions(-) diff --git a/Scripts/devInstall/Windows/DevInstall.ps1 b/Scripts/devInstall/Windows/DevInstall.ps1 index b9f43894492f..31ea6c69d31f 100644 --- a/Scripts/devInstall/Windows/DevInstall.ps1 +++ b/Scripts/devInstall/Windows/DevInstall.ps1 @@ -135,23 +135,13 @@ Function main $operation += OpCNTKMKL3 -cache $localCache -targetFolder $localDir $operation += OpSwig3010 -cache $localCache -targetFolder $localDir $operation += OpProtoBuf310VS15 -cache $localCache -targetFolder $localDir -repoDirectory $CloneDirectory + $operation += OpProtoBuf310VS15Prebuild -cache $localCache -targetFolder $localDir $operation += OpZlibVS15 -cache $localCache -targetFolder $localDir -repoDirectory $CloneDirectory + $operation += OpZlibVS15Prebuild -cache $localCache -targetFolder $localDir $operation += OpOpenCV31 -cache $localCache -targetFolder $localDir - if ($ServerLocation) { - $operation += OpProtoBuf310VS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir - $operation += OpZLibVS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir - } $operation += OpAnaconda3411 -cache $localCache -targetFolder $localDir $operation += OpAnacondaEnv34 -targetFolder $localDir -repoDir $repositoryRootDir -repoName $reponame - - #$operation += OpGit2101 -cache $localCache - #$operation += OpGitClone -targetFolder $repositoryRootDir -targetDir $reponame - #$operation += OpSysinternals -cache $localCache -targetFolder $localDir - #$operation += OpOpenCVInternal $ServerLocation -cache $localCache -targetFolder $localDir - - - $operationList = @() $operationList += (VerifyOperations $operation) diff --git a/Scripts/devInstall/Windows/helper/Action.ps1 b/Scripts/devInstall/Windows/helper/Action.ps1 index 8d7d351a2c85..521f93e74e8f 100644 --- a/Scripts/devInstall/Windows/helper/Action.ps1 +++ b/Scripts/devInstall/Windows/helper/Action.ps1 @@ -42,7 +42,6 @@ function InstallExeForPlatform( if (PlatformMatching $platform) { InstallExe $table } - return } function InstallExe( @@ -154,7 +153,6 @@ function InstallWheel( Invoke-DosCommand pip (Write-Output install $whl) -maxErrorLevel 0 $env:PATH = $oldPath - return } function InstallMSI( @@ -186,8 +184,6 @@ function MakeDirectory( New-Item $path -type directory | Out-Null } } - - return } function RobocopyFromLocalCache( @@ -219,7 +215,6 @@ function RobocopySourceDestination( $param = "$source $destination $option" DoProcess -doExecute $Execute -command $roboCopyCmd -param $param -maxErrorLevel 8 -throwOnError $true - return } function SetEnvironmentVariable( @@ -230,7 +225,7 @@ function SetEnvironmentVariable( $func = $table["Function"] $name = $table["EnvVar"] $content = $table["Content"] - $location = "Machine" + $location = "User" if (-not $Execute) { return @@ -244,7 +239,6 @@ function SetEnvironmentVariable( } SetEnvVar -name "$name" -content "$content" - return } function AddToPath( @@ -283,7 +277,6 @@ function AddToPath( if ($Execute) { SetEnvVar -name $env -content "$pathvalue" } - return } function ExtractAllFromZipForPlatform( @@ -422,8 +415,6 @@ set path="$appDir";%PATH% & tar.exe -xz --force-local -f "$destination\$targzFil } Remove-Item "$destination\$targzFileName" -ErrorAction SilentlyContinue - - return } function CreateBatch( @@ -479,7 +470,6 @@ function SetRegistryKey( } } } - return } function SetRegistryKeyNameData( @@ -594,7 +584,6 @@ function DoProcess( if (-not $throwOnError) { if ($ecode -gt $maxErrorLevel) { Write-Verbose "Running [start-process $commandString $param] failed with exit code [$ecode]" - return } return } @@ -602,7 +591,6 @@ function DoProcess( if ($ecode -gt $maxErrorLevel) { throw "Running [start-process $commandString $param] failed with exit code [$ecode]" } - return } function SetEnvVar( @@ -637,7 +625,6 @@ function RunPowershellCommand( if ($ecode -gt $maxErrorLevel) { throw "Running 'powershell.exe $commandString' failed with exit code [$ecode]" } - return } function Invoke-DosCommand { diff --git a/Scripts/devInstall/Windows/helper/Operations.ps1 b/Scripts/devInstall/Windows/helper/Operations.ps1 index fb4fac7f7c37..342328dee8ae 100644 --- a/Scripts/devInstall/Windows/helper/Operations.ps1 +++ b/Scripts/devInstall/Windows/helper/Operations.ps1 @@ -219,48 +219,47 @@ function OpProtoBuf310VS15( # the script file can be used to create the compiled protobuf libraries in $targetPath = $targetFolder\$prodSubDir $prodName = "ProtoBuf 3.1.0 Source" + $prodSrcSubdir = "protobuf-3.1.0" $prodFile = "protobuf310.zip" - $prodName = "protobuf-3.1.0" $prodSubDir = "protobuf-3.1.0-vs15" $batchFile = "buildProtoVS15.cmd" $protoSourceDir = join-path $targetFolder "src" - $completeProtoSourceDir = join-Path $protoSourceDir $prodName + $targetPath = Join-Path $protoSourceDir $prodSrcSubdir $scriptDirectory = join-path $targetFolder "script" - $targetPath = join-path $targetFolder $prodSubDir - $envVar = "PROTOBUF_PATH" - $envValue = $targetPath + $buildDir = join-path $targetFolder $prodSubDir $downloadSource = "https://github.com/google/protobuf/archive/v3.1.0.zip" $downloadSize = 5648581 @( @{ShortName = "PROTO310VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; - Verification = @( @{Function = "VerifyDirectory"; Path = $completeProtoSourceDir } ); + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize} ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $protoSourceDir; zipSubTree = $prodName; destinationFolder = $prodName }, + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $protoSourceDir; zipSubTree = $prodSrcSubdir; destinationFolder = $prodSrcSubdir }, @{Function = "MakeDirectory"; Path = $scriptDirectory }, - @{Function = "CreateBuildProtobufBatch"; FileName = "$scriptDirectory\$batchFile"; SourceDir = $completeProtoSourceDir; TargetDir = $targetPath; RepoDirectory = $repoDirectory } ); + @{Function = "CreateBuildProtobufBatch"; FileName = "$scriptDirectory\$batchFile"; SourceDir = $targetPath; TargetDir = $buildDir; RepoDirectory = $repoDirectory } ); } ) } -function OpProtoBuf310VS15Internal( - [parameter(Mandatory=$true)][string] $server, +function OpProtoBuf310VS15Prebuild( [parameter(Mandatory=$true)][string] $cache, [parameter(Mandatory=$true)][string] $targetFolder) { - $prodName = "ProtoBuf 3.1.0 Prebuild VS2015" - $prodFile = "PreBuildProtobuf310vs15.zip" + $prodName = "ProtoBuf 3.1.0 VS15 CNTK Prebuild" + $prodFile = "protobuf-3.1.0-vs15.zip" $prodSubDir = "protobuf-3.1.0-vs15" + $targetPath = join-path $targetFolder $prodSubDir $envVar = "PROTOBUF_PATH" $envValue = $targetPath - $downloadSource = "$server\$prodFile" + $downloadSource = "https://cntk.ai/binarydrop/prerequisites/protobuf/protobuf-3.1.0-vs15.zip" + $downloadSize = 0 @( @{ShortName = "PROTO310VS15PRE"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); - Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); - Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree ="protobuf-3.1.0-vs15"; destinationFolder = $prodSubDir }, - @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize} ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree = $prodSubDir; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); } ) } @@ -325,7 +324,8 @@ function OpZlibVS15( $prodName = "zlib / libzip from source" $zlibProdName = "zlib-1.2.8" $zlibFilename = "zlib128.zip" - $zlibDownloadSource= "https://netix.dl.sourceforge.net/project/libpng/zlib/1.2.8/zlib128.zip" + # $zlibDownloadSource = "https://netix.dl.sourceforge.net/project/libpng/zlib/1.2.8/zlib128.zip" + $zlibDownloadSource = "https://cntk.ai/binarydrop/prerequisites/zip/zlib128.zip" $downloadSizeZlib = 0 $libzipProdName = "libzip-1.1.3" @@ -342,7 +342,7 @@ function OpZlibVS15( $envVar = "ZLIB_PATH" $envValue = $targetPath - @( @{ShortName = "ZLIBVS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + @( @{ShortName = "ZLIBVS15"; VerifyInfo = "Checking for $prodName in $sourceCodeDir"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = "$sourceCodeDir\$zlibProdName" }, @{Function = "VerifyDirectory"; Path = "$sourceCodeDir\$libzipProdName" }, @{Function = "VerifyFile"; Path = "$scriptDirectory\$batchFile" } ); @@ -355,5 +355,29 @@ function OpZlibVS15( } ) } +function OpZlibVS15Prebuild( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "ZLib VS15 CNTK Prebuild" + $prodFile = "zlib-vs15.zip" + $prodSubDir = "zlib-vs15" + + + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "ZLIB_PATH" + $envValue = $targetPath + $downloadSource = "https://cntk.ai/binarydrop/prerequisites/zip/zlib-vs15.zip" + $downloadSize = 0 + + @( @{ShortName = "ZLIBVS15PRE"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize} ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree = $prodSubDir; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + } ) +} + # vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Verification.ps1 b/Scripts/devInstall/Windows/helper/Verification.ps1 index c69905f95ac4..09b87eba520f 100644 --- a/Scripts/devInstall/Windows/helper/Verification.ps1 +++ b/Scripts/devInstall/Windows/helper/Verification.ps1 @@ -298,10 +298,11 @@ function VerifyEnvironmentAndData( $func = $table["Function"] $name = $table["EnvVar"] $content = $table["Content"] - $location = "Machine" - - $envContent = [environment]::GetEnvironmentVariable($name, $location) + $location = "User" + $path = Join-Path "env:" $name + $envContent = Get-ChildItem $path -ErrorAction SilentlyContinue + $envContent = $envContent.Value $noInstallRequired = ($envContent -ieq $content) Write-Verbose "[$func]: [$name] == [$content] returned [$noInstallRequired]" From 1ba7faacc8f2ea24dcec6f5edcd2d4478d414fcf Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Tue, 10 Jan 2017 10:46:36 +0100 Subject: [PATCH 080/120] remove unused command line parameter, add AnacondaBasePath option --- Scripts/devInstall/Windows/DevInstall.ps1 | 31 ++++++++----------- .../devInstall/Windows/helper/Operations.ps1 | 13 ++++---- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/Scripts/devInstall/Windows/DevInstall.ps1 b/Scripts/devInstall/Windows/DevInstall.ps1 index 31ea6c69d31f..2ceda00ac408 100644 --- a/Scripts/devInstall/Windows/DevInstall.ps1 +++ b/Scripts/devInstall/Windows/DevInstall.ps1 @@ -18,21 +18,15 @@ .PARAMETER localCache This optional parameter can be used to specify the directory downloaded components will be stored in - .PARAMETER ServerLocation - This is an optional parameter. The script can install pre-compiled components, this parameter - specifies the location on a server where this componentents are downloaded from. - This is useful for a team environment to share the components which need to get compiled (Protobuf, Zlib, libzip) - - .PARAMETER CloneDirectory - By default the installer should be executed out of the \Scripts\devInstall\Windows directory. Out of this - location the installer computes the root directory of the CNTK clone. In the case the installer is in a different location, - the root directory of the CNTK clone can be specified using this parameter - -.EXAMPLE + .PARAMETER AnacondaBasePath + This is an optional parameter and can be used to specify an already installed Anaconda3 installation. + By default a version of Anaconda3 will be installed into [C:\local\Anaconda3-4.1.1-Windows-x86_64] + + .EXAMPLE installer.ps1 Run the installer and see what operations would be performed -.EXAMPLE + .EXAMPLE installer.ps1 -Execute Run the installer and install the development tools @@ -41,10 +35,12 @@ Run the installer, but don't install any GPU specific tools .EXAMPLE - installer.ps1 -Execute -NoGpu -CloneDirectory c:\repos\CNTKAlternate + installer.ps1 -Execute -NoGpu - Run the installer, but don't install any GPU specific tools .EXAMPLE + .\install.ps1 -Execute -AnacondaBasePath d:\mytools\Anaconda34 + + This will install Anaconda in the [d:\mytools\Anaconda34] directory, or reuse this Anaconda installation if the directory exists. #> @@ -54,8 +50,7 @@ Param( [parameter(Mandatory=$false)] [switch] $NoGpu, [parameter(Mandatory=$false)] [string] $localCache = "c:\installCacheCntk", [parameter(Mandatory=$false)] [string] $InstallLocation = "c:\local", - [parameter(Mandatory=$false)] [string] $ServerLocation, - [parameter(Mandatory=$false)] [string] $CloneDirectory) + [parameter(Mandatory=$false)] [string] $AnacondaBasePath = "C:\local\Anaconda3-4.1.1-Windows-x86_64") $roboCopyCmd = "robocopy.exe" $localDir = $InstallLocation @@ -139,8 +134,8 @@ Function main $operation += OpZlibVS15 -cache $localCache -targetFolder $localDir -repoDirectory $CloneDirectory $operation += OpZlibVS15Prebuild -cache $localCache -targetFolder $localDir $operation += OpOpenCV31 -cache $localCache -targetFolder $localDir - $operation += OpAnaconda3411 -cache $localCache -targetFolder $localDir - $operation += OpAnacondaEnv34 -targetFolder $localDir -repoDir $repositoryRootDir -repoName $reponame + $operation += OpAnaconda3411 -cache $localCache -AnacondaBasePath $AnacondaBasePath + $operation += OpAnacondaEnv34 -AnacondaBasePath $AnacondaBasePath -repoDir $repositoryRootDir -repoName $reponame $operationList = @() $operationList += (VerifyOperations $operation) diff --git a/Scripts/devInstall/Windows/helper/Operations.ps1 b/Scripts/devInstall/Windows/helper/Operations.ps1 index 342328dee8ae..6484cd50c170 100644 --- a/Scripts/devInstall/Windows/helper/Operations.ps1 +++ b/Scripts/devInstall/Windows/helper/Operations.ps1 @@ -1,10 +1,11 @@ function OpAnaconda3411( [parameter(Mandatory=$true)][string] $cache, - [parameter(Mandatory=$true)][string] $targetFolder) + [parameter(Mandatory=$true)][string] $AnacondaBasePath) { + $targetFolder = Split-Path $AnacondaBasePath -Parent + $prodSubDir = Split-Path $AnacondaBasePath -Leaf $prodName = "Anaconda3-4.1.1" $prodFile = "Anaconda3-4.1.1-Windows-x86_64.exe" - $prodSubDir = "Anaconda3-4.1.1-Windows-x86_64" $targetPath = join-path $targetFolder $prodSubDir $downloadSource = "https://repo.continuum.io/archive/Anaconda3-4.1.1-Windows-x86_64.exe" $downloadSize = 370055720 @@ -17,13 +18,13 @@ } function OpAnacondaEnv34( - [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $AnacondaBasePath, [parameter(Mandatory=$true)][string] $repoDir, [parameter(Mandatory=$true)][string] $reponame) { $prodName = "Python 3.4 Environment" - - $prodSubDir = "Anaconda3-4.1.1-Windows-x86_64" + $targetFolder = Split-Path $AnacondaBasePath -Parent + $prodSubDir = Split-Path $AnacondaBasePath -Leaf $targetPath = join-path $targetFolder $prodSubDir $envName = "cntkdev-py34" $envDir = "envs\$envName" @@ -128,7 +129,7 @@ function OpMSMPI70SDK( $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; $downloadSize = 2285568 - @( @{Name = "MSMPI SDK70 Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70 SDK"; ActionInfo = "Install MSMPI 70 SDK"; + @( @{Name = "MSMPI SDK70 Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70 SDK"; ActionInfo = "Installing MSMPI 70 SDK"; Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI SDK \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); #Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft MPI SDK \(\d+\."; Compare = "^Microsoft MPI SDK \(7\.0\.12437\.6\)"; MatchExact = $false } ); Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$localFilename"; ExpectedSize = $downloadSize } ); From 77fa2098c41117219bf8fc8ca9f425feafd54c2f Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Tue, 10 Jan 2017 11:33:54 +0100 Subject: [PATCH 081/120] changes to readme --- Scripts/devInstall/Windows/readme.md | 85 ++++++++-------------------- 1 file changed, 25 insertions(+), 60 deletions(-) diff --git a/Scripts/devInstall/Windows/readme.md b/Scripts/devInstall/Windows/readme.md index 38d2062bc556..4f344ce7ce4c 100644 --- a/Scripts/devInstall/Windows/readme.md +++ b/Scripts/devInstall/Windows/readme.md @@ -2,82 +2,48 @@ This documents describes the setup of the CNTK developer environment. The script follows overall the recipe described in the public documentation of the CNTK developer setup, available [here](https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows). ->Note: Before startingt the script base installation, please check the CNTK Wiki for more detailed instructions (?here?) +>Note: Before startingt the script base installation, please check the CNTK Wiki for more detailed instructions. Make sure the prerequisites for the installation are satisfied on your system. ### The installation script -The most up-to-date version of the installation script can be found on ``\\cntk-muc00\devinstall``. Copy -The prepared installation script to a directory on your system. In the following we assume you are -going to copy the files in to the directory `c:\local\devinstall`. Start a Windows command shell and enter -``` -c: -cd \ -md local && cd local -md devinstall && cd devinstall -robocopy \\cntk-muc00\devinstall . /MIR -``` -The installation script itself is written for the Powershell environment. By default Powershell doesn't -allow the execution of any scripts. To allow execution of the installation script, -start Powershell from a **standard Windows command shell** by: +The installation script itself is written for the Powershell environment. By default Powershell doesn't allow the execution of any scripts. To allow execution of the installation script, start Powershell from a **standard Windows command shell** by: ``` start powershell -executionpolicy remotesigned ``` -This will open a Powershell environment. In this enviroment change into the directory you copied -the install script into, here you can directly start the installation script +This will open a Powershell environment. In this enviroment change into the directory you copied the install script into, here you can directly start the installation script ``` cd c:\local\devinstall .\devinstall.ps1 ``` ->Note: If you now receive an errormessage stating that running scripts is disabled on your system, -please make sure you started powershell with the changed executionpolicy. This is enabling +>Note: If you now receive an errormessage stating that running scripts is disabled on your system, please make sure you started powershell with the changed executionpolicy. This is enabling script execution in your created Powershell environment. -The script will inspect your system and determine the components which are missing to build the -Microsoft Cognitive Toolkit. You will be -notified about the proposed installation steps. At this point you are running in a **demo** mode - -NO changes to your system are being performed. +The script will inspect your system and determine the components which are missing to build the Microsoft Cognitive Toolkit. You will be notified about the proposed installation steps. At this point you are running in a **demo** mode - NO changes to your system are being performed. ### Running the installation script -If you are satisfied with the proposed changes, you can proceed to the actual installation. -The installation script supports several command line options. ``get-help .\install.ps1`` will -give you a list over the available option. At this point it is recommended to start the installation -by adding the `-Execute` parameter: +If you are satisfied with the proposed changes, you can proceed to the actual installation. The installation script supports several command line options. ``get-help .\install.ps1`` will give you a list over the available option. At this point it is recommended to start the installation by adding the `-Execute` parameter: ``` .\devinstall.ps1 -execute ``` -The script will download needed components from the web, therefore a connection to the Internet is required. -Downloaded components are stored in the directory ``c:\cntkLocalCache`` and can be removed after complete -installation. During execution of the installation script you might receive requests/warnings from UAC -(User Account Control) depending on your system configuration. Please acknowledge the execution and -installation of the downloaded components. +The script will download needed components from the web, therefore a connection to the Internet is required. Downloaded components are stored in the directory ``c:\installCacheCntk`` and can be removed after complete installation. During execution of the installation script you might receive requests/warnings from UAC (User Account Control) depending on your system configuration. Please acknowledge the execution and installation of the downloaded components. -> Note: Some components of the installation script (i.e. NVidia CUDA install ) might performan a reboot of your -system. You can just start the installation script again. It will analyze your system again and reuse already -downloaded components. +> Note: Some components of the installation script (i.e. NVidia CUDA install ) might performan a reboot of your system. You can just start the installation script again. It will analyze your system again and reuse already downloaded components and will only perform the actions necessary. ### Result of the installation script The following changes (if necessary) to your system will be performed by the installation script: -- Installation of the Visual Studio 2012 and Visual Studio 2013 runtime modules - Installation of Microsoft MPI - Installation of the Microsoft MPI SDK -- If not present on your system. Git will be installed - - Location: ``%PROGAMFILES%\Git`` - - Environment: Added to the PATH -- NVidia CUDA 7.5 - - Location: ``%PROGRAMFILES(x86)%\NVIDIA Corporation``, ``%PROGRAMFILES%\NVIDIA GPU Computing Toolkit``, - see NVidia documentation for details - - Environment: ``CUDA_PATH``, ``CUDA_PATH_V7_5`` -- NVidia CuDNN 5.1 for CUDA 7.5 - - Location: ``c:\local\cudnn-7.5-v5.1`` +- NVidia CuDNN 5.1 for CUDA 8 + - Location: ``c:\local\cudnn-8.0-v5.1`` - Environment: ``CUDNN_PATH`` - NVidia CUB 1.4.1 - Location: ``c:\local\cub-1.4.1`` - Environment: ``CUB_PATH`` - Boost 1.60 - - Location: ``c:\local\boost_1_60_0`` + - Location: ``boost_1_60_0-msvc-14.0`` - Environment: ``BOOST_INCLUDE_PATH``, ``BOOST_LIB_PATH`` - A CNTK specific MKL library - Location: ``c:\local\cntkmkl`` @@ -85,32 +51,31 @@ The following changes (if necessary) to your system will be performed by the ins - OpenCV 3.1.0 - Location: ``c:\local\Opencv3.1.0`` - Environment: ``OPENCV_PATH_V31`` -- Zlib - Precompiled version of Zlib and libzip - - Location: ``c:\local\zlib`` +- Zlib Source + - Location: ``c:\local\src\zlib-1.2.8`` +- Libzip Source + - Location: ``c:\local\src\libzip-1.1.3`` +- Zlib - Precompiled version of Zlib and libzip for CNTK + - Location: ``c:\local\zlib-vs15`` - Environment: ``ZLIB_PATH`` -- Protobuf 3.1.0 libararies and headers - - Location: ``c:\local\protobuf-3.1.0`` +- Protobuf 3.1.0 Source + - Location: ``c:\local\src\protobuf-3.1.0`` +- Protobuf 3.1.0 - Precompiled version for CNTK + - Location: ``c:\local\protobuf-3.1.0-vs15`` - Environment: ``PROTOBUF_PATH`` - SWIG 3.0.10 - Location: ``c:\local\swigwin-3.0.10`` - Environment: ``SWIG_PATH`` -- CNTK Git repository clone - - Location: ``c:\repos\CNTK`` - Anaconda3 - 4.1.1 - Location: ``c:\local\Anaconda3-4.1.1-Windows-x86_64`` This is a local Anaconda install, it hasn't been added to the path, and is registered only to the current user ->Note: This script already installed the compiled Protobuf library, as well as the compiled -zlib and libzip components. It isn't necessary to perform the additional compilation steps (listed in the -public documentation) for these components. +>Note: This script already installed the compiled Protobuf library, as well as the compiled zlib and libzip components. It isn't necessary to perform the additional compilation steps (listed in the public documentation) for these components. Once the script finished, you should be ready to build. A couple of points to note: - - Depending on the tools installed, the script might have done changes to your system (especially a CUDA install). - Rebooting you system might be a good choice. - - It is possible to run the script multiple times. Powershell is very specific that it doesn't pick up changes to - environment variables which are done in the script, you have to restart powershell to pick up the latest environment. - - If you are planning on using a GPU, you should install the latest GPU driver from NVidia, and reboot - your system. + - Depending on the tools installed, the script might have done changes to your system. Rebooting you system might be a good choice. + - It is possible to run the script multiple times. Powershell is very specific that it doesn't pick up changes to environment variables which are done in the script, you have to restart powershell to pick up the latest environment. + - If you are planning on using a GPU, you should install the latest GPU driver from NVidia, and reboot your system. From 7493611319b1d8b380c8510a4b9100afd7c87b7e Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Tue, 10 Jan 2017 14:08:42 +0100 Subject: [PATCH 082/120] address CR comments --- Scripts/README.md | 2 +- Scripts/devInstall/Windows/DevInstall.ps1 | 66 +++++++------------ Scripts/devInstall/Windows/buildProtoVS15.bat | 51 ++++++-------- Scripts/devInstall/Windows/buildZlibVS15.bat | 61 +++++++++-------- Scripts/devInstall/Windows/helper/Display.ps1 | 4 +- .../devInstall/Windows/helper/Download.ps1 | 61 ++++++++--------- .../Windows/helper/PreRequisites.ps1 | 23 +------ .../Windows/helper/Verification.ps1 | 22 +------ 8 files changed, 109 insertions(+), 181 deletions(-) diff --git a/Scripts/README.md b/Scripts/README.md index 394bca63f0d4..2ce8be81630b 100644 --- a/Scripts/README.md +++ b/Scripts/README.md @@ -10,7 +10,7 @@ CNTK on a users system. They are NOT intended to run from this location in the r ## CNTK Development Environment Installer -The directory `devInstall` contains scripts which are used to create setup an environment to build CNTK from source. They are intended to run from this location in the repository. +The directory `devInstall` contains a Windows scripts which can be used to create an environment for building CNTK from source. The script is intended to run from its location in the repository. * `devInstall/Windows` - Create a Visual Studio based development environment on Windows diff --git a/Scripts/devInstall/Windows/DevInstall.ps1 b/Scripts/devInstall/Windows/DevInstall.ps1 index 2ceda00ac408..e3a1ecd62c77 100644 --- a/Scripts/devInstall/Windows/DevInstall.ps1 +++ b/Scripts/devInstall/Windows/DevInstall.ps1 @@ -23,31 +23,25 @@ By default a version of Anaconda3 will be installed into [C:\local\Anaconda3-4.1.1-Windows-x86_64] .EXAMPLE - installer.ps1 + .\devInstall.ps1 Run the installer and see what operations would be performed .EXAMPLE - installer.ps1 -Execute + .\devInstall.ps1 -Execute Run the installer and install the development tools -.EXAMPLE - installer.ps1 -Execute -NoGpu - Run the installer, but don't install any GPU specific tools -.EXAMPLE - installer.ps1 -Execute -NoGpu - -.EXAMPLE - .\install.ps1 -Execute -AnacondaBasePath d:\mytools\Anaconda34 + .EXAMPLE + .\devInstall.ps1 -Execute -AnacondaBasePath d:\mytools\Anaconda34 - This will install Anaconda in the [d:\mytools\Anaconda34] directory, or reuse this Anaconda installation if the directory exists. + If the directory [d:\mytools\Anaconda34] exists, the installer will assume it contains a complete Anaconda installation. + If the directory doesn't exist, Anaconda will be installed into this directory. #> [CmdletBinding()] Param( [parameter(Mandatory=$false)] [switch] $Execute, - [parameter(Mandatory=$false)] [switch] $NoGpu, [parameter(Mandatory=$false)] [string] $localCache = "c:\installCacheCntk", [parameter(Mandatory=$false)] [string] $InstallLocation = "c:\local", [parameter(Mandatory=$false)] [string] $AnacondaBasePath = "C:\local\Anaconda3-4.1.1-Windows-x86_64") @@ -56,39 +50,26 @@ $roboCopyCmd = "robocopy.exe" $localDir = $InstallLocation -# Get the current script's directory and Dot-source the a file with common Powershell script function residing in the the current script's directory +# Get the current script's directory $MyDir = Split-Path $MyInvocation.MyCommand.Definition -if ($CloneDirectory) { - $reponame = Split-Path $CloneDirectory -Leaf - $repositoryRootDir = Split-Path $CloneDirectory - - $solutionfile = join-path $clontDirectory "CNTK.SLN" - - if (-not (Test-Path -Path $solutionFile -PathType Leaf)) { - Write-Warning "[$CloneDirectory] was specified as the location of the CNTK sourcecode directory." - Write-Warning "The specified directory is not a valid clone of the CNTK Github project." - throw "Terminating install operation" - } -} -else { - $CloneDirectory = Split-Path $mydir - $CloneDirectory = Split-Path $CloneDirectory - $CloneDirectory = Split-Path $CloneDirectory +$CloneDirectory = Split-Path $mydir +$CloneDirectory = Split-Path $CloneDirectory +$CloneDirectory = Split-Path $CloneDirectory - $reponame = Split-Path $CloneDirectory -Leaf - $repositoryRootDir = Split-Path $CloneDirectory +$reponame = Split-Path $CloneDirectory -Leaf +$repositoryRootDir = Split-Path $CloneDirectory - $solutionfile = join-path $CloneDirectory "CNTK.SLN" +$solutionfile = Join-Path $CloneDirectory "CNTK.SLN" - if (-not (Test-Path -Path $solutionFile -PathType Leaf)) { - Write-Warning "The install script was started out of the [$mydir] location. Based on this" - Write-Warning "[$CloneDirectory] should be the location of the CNTK sourcecode directory." - Write-Warning "The specified directory is not a valid clone of the CNTK Github project." - throw "Terminating install operation" - } +if (-not (Test-Path -Path $solutionFile -PathType Leaf)) { + Write-Warning "The install script was started out of the [$mydir] location. Based on this" + Write-Warning "[$CloneDirectory] should be the location of the CNTK sourcecode directory." + Write-Warning "The specified directory is not a valid clone of the CNTK Github project." + throw "Terminating install operation" } + . "$MyDir\helper\Display" . "$MyDir\helper\Common" . "$MyDir\helper\Operations" @@ -116,12 +97,9 @@ Function main $operation += OpScanProgram $operation += OpCheckVS15Update3 - if (-not $NoGpu) { - $operation += OpCheckCuda8 - - $operation += OpNVidiaCudnn5180 -cache $localCache -targetFolder $localDir - $operation += OpNvidiaCub141 -cache $localCache -targetFolder $localDir - } + $operation += OpCheckCuda8 + $operation += OpNVidiaCudnn5180 -cache $localCache -targetFolder $localDir + $operation += OpNvidiaCub141 -cache $localCache -targetFolder $localDir $operation += OpCMake362 -cache $localCache $operation += OpMSMPI70 -cache $localCache diff --git a/Scripts/devInstall/Windows/buildProtoVS15.bat b/Scripts/devInstall/Windows/buildProtoVS15.bat index ac560e51983e..b389f6d5b12b 100644 --- a/Scripts/devInstall/Windows/buildProtoVS15.bat +++ b/Scripts/devInstall/Windows/buildProtoVS15.bat @@ -10,40 +10,38 @@ if /I "%CMDCMDLINE%" neq ""%COMSPEC%" " ( @echo. exit /b 0 ) +setlocal +if "%~1"=="" goto HELP +if "%~1"=="-?" goto HELP +if /I "%~1"=="-h" goto HELP +if /I "%~1"=="-help" goto HELP +if "%~2"=="" goto HELP +if not "%~3"=="" goto HELP -if "%1"=="" ( goto HELP ) -if "%1"=="-?" ( goto HELP ) -if /I "%1"=="-h" ( goto HELP ) -if /I "%1"=="-help" ( goto HELP ) -if "%2"=="" ( goto HELP ) -if not "%3"=="" ( goto HELP ) +set SOURCEDIR=%~f1 +set TARGETDIR=%~f2 -SET SOURCEDIR=%~f1 -SET TARGETDIR=%~f2 - -IF "%SOURCEDIR:~-1%"=="\" SET SOURCEDIR=%SOURCEDIR:~,-1% -IF "%TARGETDIR:~-1%"=="\" SET TARGETDIR=%TARGETDIR:~,-1% +if "%SOURCEDIR:~-1%"=="\" set SOURCEDIR=%SOURCEDIR:~,-1% +if "%TARGETDIR:~-1%"=="\" set TARGETDIR=%TARGETDIR:~,-1% if not exist "%SOURCEDIR%\build" ( @echo Error: "%SOURCEDIR%" is not a valid ProtoBuf source directory goto FIN ) -set cmakePath= -for %%f in ("cmake.exe") do set cmakePath=%%~$PATH:f -if "%cmakePath%"=="" ( +where -q cmake.exe +if errorlevel 1 ( @echo Error: CMAKE.EXE not found in PATH! goto FIN ) -SET VCDIRECTORY=%VS140COMNTOOLS% -if "%VCDIRECTORY%"=="" ( +if not defined %VS140COMNTOOLS% ( @echo Environment variable VS140COMNTOOLS not defined. @echo Make sure Visual Studion 2015 Update 3 is installed. goto FIN ) - -IF "%VCDIRECTORY:~-1%"=="\" SET VCDIRECTORY=%VCDIRECTORY:~,-1% +set VCDIRECTORY=%VS140COMNTOOLS% +if "%VCDIRECTORY:~-1%"=="\" set VCDIRECTORY=%VCDIRECTORY:~,-1% if not exist "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" ( echo Error: "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" not found. @@ -52,8 +50,8 @@ if not exist "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" ( ) @echo. -@echo This will build Protobuf-3.1.0 using Visual Studio 2015 -@echo ------------------------------------------------------- +@echo This will build Protobuf for CNTK using Visual Studio 2015 +@echo ---------------------------------------------------------- @echo The configured settings for the batch file: @echo Visual Studio directory: %VCDIRECTORY% @echo Protobuf source directory: %SOURCEDIR% @@ -82,27 +80,20 @@ cd .. popd -setx PROTOBUF_PATH %TARGETDIR% -set PROTOBUF_PATH=%TARGETDIR% - goto FIN - - :HELP @echo. @echo Use this script to build the Protobuf library for CNTK. @echo The script requires two parameter @echo Parameter 1: The complete path to the ProtoBuf source directory -@echo i.e: C:\local\src\protobuf-3.1.0 +@echo e.g. C:\local\src\protobuf-3.1.0 @echo Parameter 2: The target path for the created binaries -@echo i.e. C:\local\protobuf-3.1.0-vs15 -@echo The sript will also set the environment variable PROTOBUF_PATH to -@echo the target directory +@echo e.g. C:\local\protobuf-3.1.0-vs15 @echo. goto FIN :FIN - +endlocal REM vim:set expandtab shiftwidth=2 tabstop=2: diff --git a/Scripts/devInstall/Windows/buildZlibVS15.bat b/Scripts/devInstall/Windows/buildZlibVS15.bat index 3f85cc8117a6..41f1b7e971b9 100644 --- a/Scripts/devInstall/Windows/buildZlibVS15.bat +++ b/Scripts/devInstall/Windows/buildZlibVS15.bat @@ -1,9 +1,10 @@ @REM Copyright (c) Microsoft. All rights reserved. @REM Licensed under the MIT license. See LICENSE.md file in the project root for full license information. @REM -@REM batch script to build image compression libraries for CNTK +@REM batch script to build the compression library used by the CNTK image reader @echo off + if /I "%CMDCMDLINE%" neq ""%COMSPEC%" " ( @echo. @echo Please execute this script from inside a regular Windows command prompt. @@ -11,21 +12,23 @@ if /I "%CMDCMDLINE%" neq ""%COMSPEC%" " ( exit /b 0 ) -if "%1"=="" ( goto HELP ) -if "%1"=="-?" ( goto HELP ) -if /I "%1"=="-h" ( goto HELP ) -if /I "%1"=="-help" ( goto HELP ) -if "%2"=="" ( goto HELP ) -if "%3"=="" ( goto HELP ) -if not "%4"=="" ( goto HELP ) +setlocal + +if "%~1"=="" goto HELP +if "%~1"=="-?" goto HELP +if /I "%~1"=="-h" goto HELP +if /I "%~1"=="-help" goto HELP +if "%~2"=="" goto HELP +if "%~3"=="" goto HELP +if not "%~4"=="" goto HELP -SET LIBZIPSOURCEDIR=%~f1 +set LIBZIPSOURCEDIR=%~f1 set ZLIBSOURCEDIR=%~f2 -SET TARGETDIR=%~f3 +set TARGETDIR=%~f3 -IF "%LIBZIPSOURCEDIR:~-1%"=="\" SET LIBZIPSOURCEDIR=%LIBZIPSOURCEDIR:~,-1% -IF "%ZLIBSOURCEDIR:~-1%"=="\" SET ZLIBSOURCEDIR=%ZLIBSOURCEDIR:~,-1% -IF "%TARGETDIR:~-1%"=="\" SET TARGETDIR=%TARGETDIR:~,-1% +if "%LIBZIPSOURCEDIR:~-1%"=="\" set LIBZIPSOURCEDIR=%LIBZIPSOURCEDIR:~,-1% +if "%ZLIBSOURCEDIR:~-1%"=="\" set ZLIBSOURCEDIR=%ZLIBSOURCEDIR:~,-1% +if "%TARGETDIR:~-1%"=="\" set TARGETDIR=%TARGETDIR:~,-1% if not exist "%LIBZIPSOURCEDIR%\CMakeLists.txt" ( @echo Error: "%LIBZIPSOURCEDIR%" not a valid LibZib directory @@ -35,15 +38,19 @@ if not exist "%ZLIBSOURCEDIR%\CMakeLists.txt" ( @echo Error: "%ZLIBSOURCEDIR%" not a valid ZLib directory goto :FIN ) +where -q cmake.exe +if errorlevel 1 ( + @echo Error: CMAKE.EXE not found in PATH! + goto FIN +) -SET VCDIRECTORY=%VS140COMNTOOLS% -if "%VCDIRECTORY%"=="" ( +if not defined %VS140COMNTOOLS% ( @echo Environment variable VS140COMNTOOLS not defined. @echo Make sure Visual Studion 2015 Update 3 is installed. goto FIN ) - -IF "%VCDIRECTORY:~-1%"=="\" SET VCDIRECTORY=%VCDIRECTORY:~,-1% +set VCDIRECTORY=%VS140COMNTOOLS% +if "%VCDIRECTORY:~-1%"=="\" set VCDIRECTORY=%VCDIRECTORY:~,-1% if not exist "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" ( echo Error: "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" not found. @@ -51,11 +58,11 @@ if not exist "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" ( goto FIN ) -@SET CMAKEGEN="Visual Studio 14 2015 Win64" +@set CMAKEGEN="Visual Studio 14 2015 Win64" @echo. -@echo This will build compression libraries for CNTK using Visual Studio 2015 -@echo ----------------------------------------------------------------------- +@echo This will build cthe compression library used by the CNTK image reader +@echo ---------------------------------------------------------------------- @echo The configured settings for the batch file: @echo Visual Studio directory: %VCDIRECTORY% @echo CMake Generator: %CMAKEGEN% @@ -83,26 +90,22 @@ call "%VCDIRECTORY%\..\..\VC\vcvarsall.bat" amd64 @cmake -DBUILD_TYPE=Release -P cmake_install.cmake @popd -setx ZLIB_PATH %TARGETDIR% -set ZLIB_PATH=%TARGETDIR% - goto FIN :HELP @echo. -@echo Use this script to build the image compression libraries for CNTK. +@echo Use this script to build the the compression library used by the CNTK image reader @echo The script requires three parameter @echo Parameter 1: The complete path to the LibZip source directory -@echo i.e: C:\local\src\libzip-1.1.3 +@echo e.g C:\local\src\libzip-1.1.3 @echo Parameter 1: The complete path to the ZLib source directory -@echo i.e: C:\local\src\zlib-1.2.8 +@echo e.g C:\local\src\zlib-1.2.8 @echo Parameter 2: The target path for the created binaries -@echo i.e. C:\local\zlib-vs15 -@echo The sript will also set the environment variable ZLIB_PATH to -@echo the target directory +@echo e.g C:\local\zlib-vs15 @echo. goto FIN :FIN +endlocal REM vim:set expandtab shiftwidth=2 tabstop=2: diff --git a/Scripts/devInstall/Windows/helper/Display.ps1 b/Scripts/devInstall/Windows/helper/Display.ps1 index 49f2c5c93a04..27e84355a908 100644 --- a/Scripts/devInstall/Windows/helper/Display.ps1 +++ b/Scripts/devInstall/Windows/helper/Display.ps1 @@ -20,7 +20,7 @@ function DisplayStartMessage { " -This script will setup the CNTK v2 Development Environment on your machine. +This script will setup the CNTK Development Environment on your machine. More help is given by calling get-help .\devInstall.ps1 The script will analyse your machine and will determine which components are required. @@ -110,7 +110,7 @@ function DisplayStart() Function DisplayEnd() { - + # empty by design } function DisplayAfterVerify( diff --git a/Scripts/devInstall/Windows/helper/Download.ps1 b/Scripts/devInstall/Windows/helper/Download.ps1 index 56ae8fe8b3eb..911f8d5c404b 100644 --- a/Scripts/devInstall/Windows/helper/Download.ps1 +++ b/Scripts/devInstall/Windows/helper/Download.ps1 @@ -24,8 +24,6 @@ function DownloadItem( Write-Verbose "Calling Operation: [$func]" $result = Invoke-Expression $expr - - return } function DownloadForPlatform( @@ -63,7 +61,6 @@ function Download( else { DownloadFileWebClient -SourceFile $source -OutFile $destination } - return } function LocalCopyFile( @@ -90,7 +87,6 @@ function LocalCopyFile( Write-Host Copying [$source] to local disk ... new-item $destination -type File -Force -ErrorAction SilentlyContinue copy-Item $source $destination -Force -ErrorAction SilentlyContinue - return } function RobocopyFromServer( @@ -195,7 +191,6 @@ function DownloadFileWebClient( [int] $timeout = 600, [int] $maxtry = 5) { - # the scriptblock to invoke a web-request $sb ={ param([string]$uri,[string]$outfile) (New-Object System.Net.WebClient).DownloadFile($uri,$outfile) @@ -241,43 +236,43 @@ function DownloadFileWebClient( switch ($jState) { "COMPLETED" { - if ($jError.Count -eq 0) { - Write-Verbose "End binary download!" + if ($jError.Count -eq 0) { + Write-Verbose "End binary download!" - Remove-Job $job -force -ErrorAction SilentlyContinue + Remove-Job $job -force -ErrorAction SilentlyContinue - # we now have the temporary file, we need to rename it - new-item $outFile -type File -Force -ErrorAction SilentlyContinue - move-Item $TempFile $OutFile -Force -ErrorAction SilentlyContinue + # we now have the temporary file, we need to rename it + new-item $outFile -type File -Force -ErrorAction SilentlyContinue + move-Item $TempFile $OutFile -Force -ErrorAction SilentlyContinue - if (Test-Path -Path $OutFile) { - # we have a file with our expected filename, we are in good shape and ready to report success - # in case the above rename failed, but the target file exist, we clean up a possible dangling TempFile - # no need to check if this file is really there. we don't care if it succeeds or not - Remove-Item -path $TempFile -ErrorAction SilentlyContinue + if (Test-Path -Path $OutFile) { + # we have a file with our expected filename, we are in good shape and ready to report success + # in case the above rename failed, but the target file exist, we clean up a possible dangling TempFile + # no need to check if this file is really there. we don't care if it succeeds or not + Remove-Item -path $TempFile -ErrorAction SilentlyContinue - return - } + return + } - # we got here because we finished the job, but some operation failed (i.e. the rename above. we can just try again) - Write-Verbose "Job completed but rename operation failed, retrying..." - continue - } + # we got here because we finished the job, but some operation failed (i.e. the rename above. we can just try again) + Write-Verbose "Job completed but rename operation failed, retrying..." + continue + } - Write-Host "Job Completed with Error: [$jStart] to [$current]" - Write-Host $jError - } + Write-Host "Job Completed with Error: [$jStart] to [$current]" + Write-Host $jError + } "RUNNING" { - Write-Host "Job Timeout: [$jStart] to [$current]" - } + Write-Host "Job Timeout: [$jStart] to [$current]" + } "FAILED" { - $current = Get-Date - Write-Host "Job Failed: [$jStart] to [$current]" - Write-Host "Error: $jError" - } + $current = Get-Date + Write-Host "Job Failed: [$jStart] to [$current]" + Write-Host "Error: $jError" + } default { - Write-Host "Job State: [$Error] - [$jStart] to [$current]" - } + Write-Host "Job State: [$Error] - [$jStart] to [$current]" + } } Remove-Job $job -force -ErrorAction SilentlyContinue } diff --git a/Scripts/devInstall/Windows/helper/PreRequisites.ps1 b/Scripts/devInstall/Windows/helper/PreRequisites.ps1 index 259f48081904..e88973184251 100644 --- a/Scripts/devInstall/Windows/helper/PreRequisites.ps1 +++ b/Scripts/devInstall/Windows/helper/PreRequisites.ps1 @@ -34,8 +34,7 @@ function PrereqInfo2013Up5( Write-Host " -We require the installation of Visual Studio 2013 Update 5 to continue. -Please follow the information here [https://support.microsoft.com/en-us/kb/3021976]. +We require the installation of Visual Studio 2013 Update 5 to continue. " } @@ -60,25 +59,7 @@ function PrereqInfoCuda8( FunctionIntro $table Write-Host " -Installation of NVidia Cuda 8.0 is a pre-requisite before installation -can continue. Please check https://developer.nvidia.com/cuda-downloads] for more -details on Cuda download. Please also remember to set the environment variable -CUDA_PATH_V8_0 to the Cuda8 installation location. -" -return -} - -function PrereqInfoCuDnn51( - [Parameter(Mandatory = $true)][hashtable] $table -) -{ - FunctionIntro $table - Write-Host " - -Installation of NVidia CuDNN 4.1 for Cuda 8.0 is a pre-requisite before installation -can continue. Please check https://developer.nvidia.com/cudn] for more -details on Cuda download. Please also remember to set the environment variable -CUDNN_PATH to the CudDnn folder in your installation location. +Installation of NVidia CUDA 8.0 is a pre-requisite before installation can continue. " return } diff --git a/Scripts/devInstall/Windows/helper/Verification.ps1 b/Scripts/devInstall/Windows/helper/Verification.ps1 index 09b87eba520f..f91fd8bfd46f 100644 --- a/Scripts/devInstall/Windows/helper/Verification.ps1 +++ b/Scripts/devInstall/Windows/helper/Verification.ps1 @@ -11,7 +11,7 @@ function VerifyOperations( foreach ($item in $verificationList) { $needsInstall = $false - Write-Host $item.VerifyInfo + Write-Host $item.VerifyInfo foreach ($verificationItem in $item.Verification) { $needsInstall = VerifyItem $verificationItem if (-not $needsInstall) { @@ -119,26 +119,6 @@ function VerifyWinProductVersion( return $noInstallRequired } -function VerifyInstallationContent( - [Parameter(Mandatory = $true)][hashtable] $table) -{ - FunctionIntro $table - $func = $table["Function"] - $path = $table["Path"] - - $noInstallRequired = (join-path $path cntk\cntk.exe | test-path -PathType Leaf) - $noInstallRequired = (join-path $path prerequisites\VS2012\vcredist_x64.exe | test-path -PathType Leaf) -and $noInstallRequired - $noInstallRequired = (join-path $path prerequisites\VS2013\vcredist_x64.exe | test-path -PathType Leaf) -and $noInstallRequired - $noInstallRequired = (join-path $path prerequisites\MSMpiSetup.exe | test-path -PathType Leaf) -and $noInstallRequired - - if ($noInstallRequired) { - Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" - return $noInstallRequired - } - - throw "`nFatal Error: Files from CNTK binary download package are missing!`nThe install script must be run out of the unpacked binary CNTK package, not from a CNTK source clone." -} - function VerifyDirectory( [Parameter(Mandatory = $true)][hashtable] $table) { From 172293b54dd7008b80c8cd051402c0c75ae35019 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Tue, 10 Jan 2017 14:46:46 +0100 Subject: [PATCH 083/120] moved to tools directory, minor bug fix in build-batch files --- Scripts/README.md | 6 ------ Tools/README.md | 7 +++++++ {Scripts => Tools}/devInstall/Windows/DevInstall.ps1 | 0 .../readme.md => Tools/devInstall/Windows/README.md | 0 {Scripts => Tools}/devInstall/Windows/buildProtoVS15.bat | 2 +- {Scripts => Tools}/devInstall/Windows/buildZlibVS15.bat | 2 +- {Scripts => Tools}/devInstall/Windows/helper/Action.ps1 | 4 ++-- {Scripts => Tools}/devInstall/Windows/helper/Common.ps1 | 0 {Scripts => Tools}/devInstall/Windows/helper/Display.ps1 | 0 {Scripts => Tools}/devInstall/Windows/helper/Download.ps1 | 0 .../devInstall/Windows/helper/Operations.ps1 | 0 .../devInstall/Windows/helper/PreRequisites.ps1 | 0 .../devInstall/Windows/helper/Verification.ps1 | 0 13 files changed, 11 insertions(+), 10 deletions(-) create mode 100644 Tools/README.md rename {Scripts => Tools}/devInstall/Windows/DevInstall.ps1 (100%) rename Scripts/devInstall/Windows/readme.md => Tools/devInstall/Windows/README.md (100%) rename {Scripts => Tools}/devInstall/Windows/buildProtoVS15.bat (98%) rename {Scripts => Tools}/devInstall/Windows/buildZlibVS15.bat (98%) rename {Scripts => Tools}/devInstall/Windows/helper/Action.ps1 (99%) rename {Scripts => Tools}/devInstall/Windows/helper/Common.ps1 (100%) rename {Scripts => Tools}/devInstall/Windows/helper/Display.ps1 (100%) rename {Scripts => Tools}/devInstall/Windows/helper/Download.ps1 (100%) rename {Scripts => Tools}/devInstall/Windows/helper/Operations.ps1 (100%) rename {Scripts => Tools}/devInstall/Windows/helper/PreRequisites.ps1 (100%) rename {Scripts => Tools}/devInstall/Windows/helper/Verification.ps1 (100%) diff --git a/Scripts/README.md b/Scripts/README.md index 2ce8be81630b..d34fab3f35cd 100644 --- a/Scripts/README.md +++ b/Scripts/README.md @@ -8,12 +8,6 @@ CNTK on a users system. They are NOT intended to run from this location in the r * `install/windows` - A script for installing a Windows CNTK *binary* drop, cf. [here](https://github.com/Microsoft/CNTK/wiki/Setup-Windows-Binary-Script). * `install/linux` - A script for installing a Linux CNTK *binary* drop, cf. [here](https://github.com/Microsoft/CNTK/wiki/Setup-Linux-Binary-Script). -## CNTK Development Environment Installer - -The directory `devInstall` contains a Windows scripts which can be used to create an environment for building CNTK from source. The script is intended to run from its location in the repository. - -* `devInstall/Windows` - Create a Visual Studio based development environment on Windows - ## CNTK Text format Converters Two Python Scripts for converting Data to CNTK Text format for using as an input for CNTK Text Format diff --git a/Tools/README.md b/Tools/README.md new file mode 100644 index 000000000000..684df7bfd61c --- /dev/null +++ b/Tools/README.md @@ -0,0 +1,7 @@ + +## CNTK Development Environment Installer + +The directory `devInstall` contains a Windows scripts which can be used to create an environment for building CNTK from source. The script is intended to run from its location in the repository. + +* `devInstall/Windows` - Create a Visual Studio based development environment on Windows + diff --git a/Scripts/devInstall/Windows/DevInstall.ps1 b/Tools/devInstall/Windows/DevInstall.ps1 similarity index 100% rename from Scripts/devInstall/Windows/DevInstall.ps1 rename to Tools/devInstall/Windows/DevInstall.ps1 diff --git a/Scripts/devInstall/Windows/readme.md b/Tools/devInstall/Windows/README.md similarity index 100% rename from Scripts/devInstall/Windows/readme.md rename to Tools/devInstall/Windows/README.md diff --git a/Scripts/devInstall/Windows/buildProtoVS15.bat b/Tools/devInstall/Windows/buildProtoVS15.bat similarity index 98% rename from Scripts/devInstall/Windows/buildProtoVS15.bat rename to Tools/devInstall/Windows/buildProtoVS15.bat index b389f6d5b12b..8fa6979a61c9 100644 --- a/Scripts/devInstall/Windows/buildProtoVS15.bat +++ b/Tools/devInstall/Windows/buildProtoVS15.bat @@ -35,7 +35,7 @@ if errorlevel 1 ( goto FIN ) -if not defined %VS140COMNTOOLS% ( +if not defined VS140COMNTOOLS ( @echo Environment variable VS140COMNTOOLS not defined. @echo Make sure Visual Studion 2015 Update 3 is installed. goto FIN diff --git a/Scripts/devInstall/Windows/buildZlibVS15.bat b/Tools/devInstall/Windows/buildZlibVS15.bat similarity index 98% rename from Scripts/devInstall/Windows/buildZlibVS15.bat rename to Tools/devInstall/Windows/buildZlibVS15.bat index 41f1b7e971b9..c70aa58a4d05 100644 --- a/Scripts/devInstall/Windows/buildZlibVS15.bat +++ b/Tools/devInstall/Windows/buildZlibVS15.bat @@ -44,7 +44,7 @@ if errorlevel 1 ( goto FIN ) -if not defined %VS140COMNTOOLS% ( +if not defined VS140COMNTOOLS ( @echo Environment variable VS140COMNTOOLS not defined. @echo Make sure Visual Studion 2015 Update 3 is installed. goto FIN diff --git a/Scripts/devInstall/Windows/helper/Action.ps1 b/Tools/devInstall/Windows/helper/Action.ps1 similarity index 99% rename from Scripts/devInstall/Windows/helper/Action.ps1 rename to Tools/devInstall/Windows/helper/Action.ps1 index 521f93e74e8f..f1ed9a39a5f9 100644 --- a/Scripts/devInstall/Windows/helper/Action.ps1 +++ b/Tools/devInstall/Windows/helper/Action.ps1 @@ -695,7 +695,7 @@ function GetBatchBuildProtoBuf( [string] $targetDir, [string] $repoDirectory) { - $batchFile = join-path $repoDirectory "Scripts\devInstall\Windows\buildProtoVS15.bat" + $batchFile = join-path $repoDirectory "tools\devInstall\Windows\buildProtoVS15.bat" @" call $batchFile $sourceDir $targetDir "@ @@ -707,7 +707,7 @@ function GetBatchBuildZlibBuf( [string] $targetDir, [string] $repoDirectory) { - $batchFile = join-path $repoDirectory "Scripts\devInstall\Windows\buildZlibVS15.bat" + $batchFile = join-path $repoDirectory "tools\devInstall\Windows\buildZlibVS15.bat" @" call $batchFile $libzipSourceDir $zlibSourceDir $targetDir diff --git a/Scripts/devInstall/Windows/helper/Common.ps1 b/Tools/devInstall/Windows/helper/Common.ps1 similarity index 100% rename from Scripts/devInstall/Windows/helper/Common.ps1 rename to Tools/devInstall/Windows/helper/Common.ps1 diff --git a/Scripts/devInstall/Windows/helper/Display.ps1 b/Tools/devInstall/Windows/helper/Display.ps1 similarity index 100% rename from Scripts/devInstall/Windows/helper/Display.ps1 rename to Tools/devInstall/Windows/helper/Display.ps1 diff --git a/Scripts/devInstall/Windows/helper/Download.ps1 b/Tools/devInstall/Windows/helper/Download.ps1 similarity index 100% rename from Scripts/devInstall/Windows/helper/Download.ps1 rename to Tools/devInstall/Windows/helper/Download.ps1 diff --git a/Scripts/devInstall/Windows/helper/Operations.ps1 b/Tools/devInstall/Windows/helper/Operations.ps1 similarity index 100% rename from Scripts/devInstall/Windows/helper/Operations.ps1 rename to Tools/devInstall/Windows/helper/Operations.ps1 diff --git a/Scripts/devInstall/Windows/helper/PreRequisites.ps1 b/Tools/devInstall/Windows/helper/PreRequisites.ps1 similarity index 100% rename from Scripts/devInstall/Windows/helper/PreRequisites.ps1 rename to Tools/devInstall/Windows/helper/PreRequisites.ps1 diff --git a/Scripts/devInstall/Windows/helper/Verification.ps1 b/Tools/devInstall/Windows/helper/Verification.ps1 similarity index 100% rename from Scripts/devInstall/Windows/helper/Verification.ps1 rename to Tools/devInstall/Windows/helper/Verification.ps1 From d5fe9a040d792684f259cc33f74e706fe30dc5fb Mon Sep 17 00:00:00 2001 From: Deyu Fu Date: Tue, 10 Jan 2017 10:15:55 -0800 Subject: [PATCH 084/120] update to fix window build error adding cast to fix Source/Math/GPUTensor.cu(728): warning C4244: '*=' : conversion from 'unsigned __int64' to 'int32_t', possible loss of data --- Source/Math/GPUTensor.cu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/Math/GPUTensor.cu b/Source/Math/GPUTensor.cu index 7a7ff5975cb1..f7b95c2591d4 100644 --- a/Source/Math/GPUTensor.cu +++ b/Source/Math/GPUTensor.cu @@ -725,7 +725,7 @@ static void LaunchTensorOpWithReduction(ElemType beta, array point C_size_t stride = 1; for (C_size_t k = 0; k < reducingOpDimVector.size(); ++k) { reducingOpDimDivmodVector.push_back(fast_divmod(stride)); - stride *= reducingOpDimVector[k]; + stride *= (C_size_t) reducingOpDimVector[k]; } FixedArray regularOpStrides(regularOpStrideVector); From bf994056c90d05ffc756c08554ad062e6ab7fde5 Mon Sep 17 00:00:00 2001 From: Chris Basoglu Date: Tue, 10 Jan 2017 10:47:54 -0800 Subject: [PATCH 085/120] Update README.md --- README.md | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 04e3047e1009..b0b7bfa33cd8 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,15 @@ # Latest news +***2017-01-10.*** CNTK for Windows supports Visual 2015 +If you pull or merge the master branch, CNTK will now require Visual Studio 2015 to build on Windows. There are two ways to move your development environment to Visual Studio 2015: + +[Migrate VS2013 to VS2015](https://github.com/Microsoft/CNTK/wiki/Setup-Migrate-VS13-to-VS15): +This gives you a fine grained control over where components are installed + +[Script driven setup](https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-with-script-on-Windows): +This gives you an mostly automated migration to Visual Studio 2015 + ***2016-12-22.*** V 2.0 Beta 7 Release Highlights of this Release: @@ -51,17 +60,6 @@ Highlights of this Release: See more in the [Release Notes](https://github.com/Microsoft/CNTK/wiki/CNTK_2_0_beta_4_Release_Notes) Get the Release from the [CNTK Releases page](https://github.com/Microsoft/CNTK/releases) -***2016-11-11.*** V 2.0 Beta 3 Release -Highlights of this Release: - -* Integration with [NVIDIA NCCL](https://github.com/NVIDIA/nccl). Works with Linux when building CNTK from sources. See here [how to enable](https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Linux#optional-nccl) -* The first V.2.0 Prerelease Nuget Package for CNTK Evaluation library -* Stability Improvements and bug fixes - -See more in the [Release Notes](https://github.com/Microsoft/CNTK/wiki/CNTK_2_0_beta_3_Release_Notes) -Get the Release from the [CNTK Releases page](https://github.com/Microsoft/CNTK/releases) - -See [all news](https://github.com/Microsoft/CNTK/wiki/News). # What is The Microsoft Cognitive Toolkit From 1222c3188edcfa118368f646bd8a60c0f644ec45 Mon Sep 17 00:00:00 2001 From: Chris Basoglu Date: Tue, 10 Jan 2017 11:40:27 -0800 Subject: [PATCH 086/120] Update log format for Philly --- bindings/python/cntk/utils/progress_print.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bindings/python/cntk/utils/progress_print.py b/bindings/python/cntk/utils/progress_print.py index 787dff694792..761eae42f2af 100644 --- a/bindings/python/cntk/utils/progress_print.py +++ b/bindings/python/cntk/utils/progress_print.py @@ -155,9 +155,9 @@ def epoch_summary(self, with_metric=False): speed = samples / time_delta self.epoch_start_time = epoch_end_time if with_metric: - self.___logprint("Finished Epoch [{}]: {}loss = {:0.6f} * {}, metric = {:0.1f}% * {} {:0.3f}s ({:5.1f} samples per second)".format(self.epochs, self.tag, avg_loss, samples, avg_metric*100.0, samples, time_delta, speed)) + self.___logprint("Finished Epoch[{} of {}]: {}loss = {:0.6f} * {}, metric = {:0.1f}% * {} {:0.3f}s ({:5.1f} samples per second)".format(self.epochs, self.num_epochs, self.tag, avg_loss, samples, avg_metric*100.0, samples, time_delta, speed)) else: - self.___logprint("Finished Epoch [{}]: {}loss = {:0.6f} * {} {:0.3f}s ({:5.1f} samples per second)".format(self.epochs, self.tag, avg_loss, samples, time_delta, speed)) + self.___logprint("Finished Epoch[{} of {}]: {}loss = {:0.6f} * {} {:0.3f}s ({:5.1f} samples per second)".format(self.epochs, self.num_epochs, self.tag, avg_loss, samples, time_delta, speed)) return avg_loss, avg_metric, samples # BUGBUG: for freq=0, we don't return anything here def ___gererate_progress_heartbeat(self): From 7a76c265330a635898c5968def193b6ea2f17ef8 Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Tue, 10 Jan 2017 13:11:55 -0800 Subject: [PATCH 087/120] typo fix --- bindings/csharp/Swig/cntk_cs.i | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index 1c8c4af4337c..00483dcc6cea 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -259,7 +259,7 @@ const std::wstring& name = L""); -%ignore CNTK::Unooling; +%ignore CNTK::Unpooling; %ignore CNTK::BatchNormalization(const Variable& operand, const Variable& scale, From 43181617fd03f6c081c1271f951f47a15cf27d2c Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Tue, 10 Jan 2017 18:14:52 -0800 Subject: [PATCH 088/120] fix g++ error --- Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index 8b869dfbabf9..25b0c8acc8bb 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -381,7 +381,7 @@ namespace CNTK NDShape inferredInputShape = ConvolutionOpOutputShape(PrimitiveOpType::Pooling, outputShape, unpoolingWindowShape, inputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, false, inferDimensions); if (inferredInputShape != inputShape) - RuntimeError("The shape of the unpooling operand %ls is different from the result of pooling the poolingInput argument using the provided options %ls", inputShape.AsString(), inferredInputShape.AsString()); + RuntimeError("The shape of the unpooling operand %ls is different from the result of pooling the poolingInput argument using the provided options %ls", inputShape.AsString().c_str(), inferredInputShape.AsString().c_str()); break; } From 3e1262483855141fffbc808a9616665adbbb10cb Mon Sep 17 00:00:00 2001 From: Nikos Karampatziakis Date: Tue, 10 Jan 2017 22:38:44 -0800 Subject: [PATCH 089/120] moved check from serialize to deserialize --- Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 4 ++-- bindings/python/cntk/ops/tests/kernel_test.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index 25b0c8acc8bb..0636a086ecbe 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -679,7 +679,7 @@ namespace CNTK dict[inputsKey] = std::move(inputUids); - if (m_op == PrimitiveOpType::Unpooling) + if (m_op == PrimitiveOpType::Block) { auto blockCompositeFunc = dynamic_cast(BlockComposite().get()); dict[blockFunctionCompositeKey] = blockCompositeFunc->SerializeBlockComposite(); @@ -715,7 +715,7 @@ namespace CNTK // The hard requirement that the serialization depends on is that // new op type values are only added to the end of the list, after Combine. // This also applies to other enums (DataType, VariableKind, etc.) - if (op > PrimitiveOpType::Block) + if (op > PrimitiveOpType::Unpooling) { LogicError("Unexpected op '%ls':'%u' (%s).", opKey.c_str(), diff --git a/bindings/python/cntk/ops/tests/kernel_test.py b/bindings/python/cntk/ops/tests/kernel_test.py index 85e8bf78b273..23f13f35358b 100644 --- a/bindings/python/cntk/ops/tests/kernel_test.py +++ b/bindings/python/cntk/ops/tests/kernel_test.py @@ -80,7 +80,6 @@ def test_op_convolution_without_padding(convolution_map, convolution_input, devi # this test handles convolution with asymmetric padding, in particular, with auto_padding is set to True # and the kernel shape is even @pytest.mark.skip(reason="Reference model takes too long to run causing timeout, needs further investigation") ->>>>>>> origin/master @pytest.mark.parametrize("input_size, conv_size, result", ASYM_CONVOLUTION_DATA) def test_asym_convolution(input_size, conv_size, result, device_id, precision): dt = PRECISION_TO_TYPE[precision] From 434ca582e7bc2029d92a8ec28169c56a46408aa4 Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Tue, 10 Jan 2017 12:59:15 +0100 Subject: [PATCH 090/120] Examples/Image/Classification/ResNet/BrainScript: Linux: point out that stack size may need to be increased Closes: #1273 References: #1074 --- Examples/Image/Classification/ResNet/BrainScript/README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Examples/Image/Classification/ResNet/BrainScript/README.md b/Examples/Image/Classification/ResNet/BrainScript/README.md index ed9b1a64bdda..2b8da36e562f 100644 --- a/Examples/Image/Classification/ResNet/BrainScript/README.md +++ b/Examples/Image/Classification/ResNet/BrainScript/README.md @@ -1,5 +1,9 @@ # CNTK Examples: Image/Classification/ResNet +> Note: if, on Linux, you experience segmentation faults when trying to run these examples, +> please increase your stack size limit. This can be done by running `ulimit -s 65536` in your shell, +> and then running CNTK from the same session. You can also check you current limits using `ulimit -a`. + ## BrainScript ### ResNet20_CIFAR10.cntk From 6c73818bf75afc4e55e47c44e3a449ee9ef24dc1 Mon Sep 17 00:00:00 2001 From: KeDengMS Date: Wed, 11 Jan 2017 01:00:09 -0800 Subject: [PATCH 091/120] [CNTK Core] Change BatchNormalization default to CuDnn in both BrainScript and V2 C++/Python, and add fallback to cntk engine when not supported by cudnn Fix epsilon in CuDnnBatchNormalization to be consistent in forward/backward Fix BatchNormalization parameter gradient accumulation Add message on CuDnnBatchNormalization forward infer batch size limitation, and adjust CNTK_202_Language_Understanding eval minibatch size to avoid CUDA failure --- .../Image/GettingStarted/04_OneConvBN.cntk | 2 +- .../CNTK/BrainScript/CNTKCoreLib/CNTK.core.bs | 2 +- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 2 +- Source/ComputationNetworkLib/TrainingNodes.h | 33 +++++++++++++--- Source/Math/CuDnnBatchNormalization.cu | 15 ++++--- Source/Math/GPUMatrix.h | 5 ++- Tests/UnitTests/V2LibraryTests/Image.h | 4 +- .../CNTK_202_Language_Understanding.ipynb | 2 +- bindings/python/cntk/layers.py | 2 +- .../python/cntk/ops/tests/non_linear_test.py | 39 +++++++++++++++++++ 10 files changed, 85 insertions(+), 21 deletions(-) diff --git a/Examples/Image/GettingStarted/04_OneConvBN.cntk b/Examples/Image/GettingStarted/04_OneConvBN.cntk index 035c4220a85e..f27804fab409 100644 --- a/Examples/Image/GettingStarted/04_OneConvBN.cntk +++ b/Examples/Image/GettingStarted/04_OneConvBN.cntk @@ -33,7 +33,7 @@ trainNetwork = { DenseBnReluLayer {outDim} = Sequential ( LinearLayer {outDim} : - BatchNormalizationLayer {spatialRank = 1} : ReLU + BatchNormalizationLayer {spatialRank = 0} : ReLU ) model = Sequential ( diff --git a/Source/CNTK/BrainScript/CNTKCoreLib/CNTK.core.bs b/Source/CNTK/BrainScript/CNTKCoreLib/CNTK.core.bs index 6ceccb5efa7f..3a652e2a9af5 100644 --- a/Source/CNTK/BrainScript/CNTKCoreLib/CNTK.core.bs +++ b/Source/CNTK/BrainScript/CNTKCoreLib/CNTK.core.bs @@ -195,7 +195,7 @@ DelayLayer {T=1, defaultHiddenActivation=0} = BatchNormalizationLayer {spatialRank = 0, # reduce over these dims. E.g. 2 to reduce over (w,h) in a [W x H x C]-shaped input initialScale = 1, normalizationTimeConstant = 5000, blendTimeConstant = 0, - epsilon = 0.00001, useCntkEngine = true} = + epsilon = 0.00001, useCntkEngine = false} = { #normShape = _ConcatArrays (Repeat (spatialRank, 1), 0) # spatial dims get a dimension of 1 (broadcasting, while all others are inferred from input) normShape = (0:1) # TODO: Update this once we support broadcasting-style parameters. diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 7024620e7b9e..16a48da59c55 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -3105,7 +3105,7 @@ namespace CNTK double normalizationTimeConstant = 0, double blendTimeConstant = 0, double epsilon = 0.00001, - bool useCuDNNEngine = false, + bool useCuDNNEngine = true, const std::wstring& name = L""); /// Create an instance of the CNTK built-in OptimizedRNNStack operation on specified input operands diff --git a/Source/ComputationNetworkLib/TrainingNodes.h b/Source/ComputationNetworkLib/TrainingNodes.h index 49b0321354b8..01f2654194b1 100644 --- a/Source/ComputationNetworkLib/TrainingNodes.h +++ b/Source/ComputationNetworkLib/TrainingNodes.h @@ -2523,22 +2523,20 @@ class BatchNormalizationNode : public ComputationNodeNonLooping, publi sliceInputGrad, // (out) gradient for data input goes here scale, // (in) out of scale and bias, only scale is needed in gradient propagation blendFactor, // (in) smoothing weight for running stats (1=use only running stats) - *m_savedMean, *m_savedInvStdDev, // (in) saved mean/invstddev values used in ForwardProp() + *m_savedMean, *m_savedInvStdDev, // (in) saved mean/invstddev values used in ForwardProp() *m_dScale, *m_dBias); // (out) gradients for scale and bias } else if (inputIndex == 1) // derivative with respect to the scale { // Derivative with respect to the scale was precomputed during input derivative computation. Matrix& grad = Input(1)->Gradient(); - grad.SetValue(grad.GetNumRows(), grad.GetNumCols(), grad.GetDeviceId(), m_dScale->Data()); - // BUGBUG: ^^ This should add the gradient, not overwrite it. + Matrix::ScaleAndAdd(1, *m_dScale, grad); } else if (inputIndex == 2) // derivative with respect to the bias { // Derivative with respect to the bias was precomputed during input derivative computation. Matrix& grad = Input(2)->Gradient(); - grad.SetValue(grad.GetNumRows(), grad.GetNumCols(), grad.GetDeviceId(), m_dBias->Data()); - // BUGBUG: ^^ Also here, this should add the gradient, not overwrite it. + Matrix::ScaleAndAdd(1, *m_dBias, grad); } // No derivatives with respect to running mean and variance. } @@ -2588,7 +2586,7 @@ class BatchNormalizationNode : public ComputationNodeNonLooping, publi { // The current implementation requires that the gradient of the first operand/input be computed // in order to compute gradients for the bias and scale parameters (2nd and 3rd inputs) - if ((Input(1)->NeedsGradient() || Input(2)->NeedsGradient()) && !Input(0)->NeedsGradient()) + if (Environment().IsTraining() && ((Input(1)->NeedsGradient() || Input(2)->NeedsGradient()) && !Input(0)->NeedsGradient())) InvalidArgument("%ls %ls currently supports learnable scale and bias parameters only if the first input also needs gradient (i.e. is dependent on at-least one learnable parameter).", NodeName().c_str(), OperationName().c_str()); if (m_convertRunningVariancePending) @@ -2631,6 +2629,29 @@ class BatchNormalizationNode : public ComputationNodeNonLooping, publi "Please specify imageLayout=\"cudnn\" in BatchNormalization node in your NDL/BrainScript " "and make sure your input data layout is CHW", NodeName().c_str(), OperationName().c_str()); } + + if (!m_useCntkEngine) + { + // Fallback to cntk engine on CPU device if cuDnn is not available, + bool cpuDevice = (m_deviceId == CPUDEVICE); + + // or if parameters cannot be handled by cuDnn (which is needed for compatibility when changing the default to cudnn) + // In Source/Math/CuDnnBatchNormalization.cu : + // if (blendFactor != 0 && (blendFactor != 1 || expAvgFactor > 0)) + // InvalidArgument("cuDNN batch normalization engine currently supports blendTimeConstant of 0 or 1 only."); + // Check ComputeBlendFactor()/ComputeExpAvgFactor() for inferring blendFactor/expAvgFactor from m_blendTimeConst/m_normTimeConst + bool cuDnnUnsupportedParams = (m_blendTimeConst != 0 && (isfinite(m_blendTimeConst) || isfinite(m_normTimeConst))); + + if (cpuDevice || cuDnnUnsupportedParams) + { + m_useCntkEngine = true; + if (cuDnnUnsupportedParams) + { + fprintf(stderr, "\nWARNING: batch normalization falls back to cntk engine for parameters not supported by cuDnn.\n"); + } + } + } + double cudnnMinEps = 1e-5; // CUDNN_BN_MIN_EPSILON if (!m_useCntkEngine && m_epsilon < cudnnMinEps) fprintf(stderr, "\nWARNING: cuDNN batch normalization requires epsilon >= %e. Epsilon will be reset to that value.\n", cudnnMinEps); diff --git a/Source/Math/CuDnnBatchNormalization.cu b/Source/Math/CuDnnBatchNormalization.cu index 9fe1e7081292..7e088cd454e8 100644 --- a/Source/Math/CuDnnBatchNormalization.cu +++ b/Source/Math/CuDnnBatchNormalization.cu @@ -24,7 +24,8 @@ public: : Base(deviceId, inOutT, spatial, imageLayout), m_cudnn(CuDnn::Instance()), m_inOutCuDnnT(GetInOutTensor(inOutT), CuDnnTensor::GetDataType()), - m_scaleBiasCuDnnT(GetScaleBiasTensor(inOutT, spatial), CuDnnTensor::GetDataType()) + m_scaleBiasCuDnnT(GetScaleBiasTensor(inOutT, spatial), CuDnnTensor::GetDataType()), + m_cudnnEpsilon(CUDNN_BN_MIN_EPSILON) { } @@ -54,14 +55,15 @@ protected: m_inOutCuDnnT.UpdateBatchSize(in.GetNumCols()); cudnnBatchNormMode_t mode = m_spatial ? CUDNN_BATCHNORM_SPATIAL : CUDNN_BATCHNORM_PER_ACTIVATION; // cuDNN will fail with BAD_PARAM if epsilon < CUDNN_BN_MIN_EPSILON. - epsilon = max(epsilon, CUDNN_BN_MIN_EPSILON); + m_cudnnEpsilon = max(epsilon, CUDNN_BN_MIN_EPSILON); if (inferenceOnly) { assert(expAvgFactor == 0 && blendFactor == 1); savedMean.Resize(0, 0); // (these are not produced in this case) savedInvStdDev.Resize(0, 0); - CUDNN_CALL(cudnnBatchNormalizationForwardInference(*m_cudnn, mode, &C::One, &C::Zero, m_inOutCuDnnT, ptr(in), m_inOutCuDnnT, ptr(out), - m_scaleBiasCuDnnT, ptr(scale), ptr(bias), ptr(runMean), ptr(runVariance), epsilon)); + CUDNN_CALL2(cudnnBatchNormalizationForwardInference(*m_cudnn, mode, &C::One, &C::Zero, m_inOutCuDnnT, ptr(in), m_inOutCuDnnT, ptr(out), + m_scaleBiasCuDnnT, ptr(scale), ptr(bias), ptr(runMean), ptr(runVariance), m_cudnnEpsilon), + "\nProbably hitting cuDNN limit on batch size, try reducing minibatch size"); } else { @@ -69,7 +71,7 @@ protected: savedInvStdDev.Resize(runMean); CUDNN_CALL(cudnnBatchNormalizationForwardTraining(*m_cudnn, mode, &C::One, &C::Zero, m_inOutCuDnnT, ptr(in), m_inOutCuDnnT, ptr(out), m_scaleBiasCuDnnT, ptr(scale), ptr(bias), expAvgFactor, ptr(runMean), ptr(runVariance), - epsilon, ptr(savedMean), ptr(savedInvStdDev))); + m_cudnnEpsilon, ptr(savedMean), ptr(savedInvStdDev))); } } @@ -81,7 +83,7 @@ protected: cudnnBatchNormMode_t mode = m_spatial ? CUDNN_BATCHNORM_SPATIAL : CUDNN_BATCHNORM_PER_ACTIVATION; // REVIEW alexeyk: change betaParamDiff to 1 and update CNTK BN engine. CUDNN_CALL(cudnnBatchNormalizationBackward(*m_cudnn, mode, &C::One, &C::One, &C::One, &C::Zero, m_inOutCuDnnT, ptr(in), m_inOutCuDnnT, ptr(srcGrad), m_inOutCuDnnT, ptr(grad), - m_scaleBiasCuDnnT, ptr(scale), ptr(scaleGrad), ptr(biasGrad), CUDNN_BN_MIN_EPSILON, ptr(savedMean), ptr(savedInvStdDev))); + m_scaleBiasCuDnnT, ptr(scale), ptr(scaleGrad), ptr(biasGrad), m_cudnnEpsilon, ptr(savedMean), ptr(savedInvStdDev))); } private: @@ -123,6 +125,7 @@ private: CuDnn::ptr_t m_cudnn; CuDnnTensor m_inOutCuDnnT; CuDnnTensor m_scaleBiasCuDnnT; + double m_cudnnEpsilon; }; template class CuDnnBatchNormEngine; diff --git a/Source/Math/GPUMatrix.h b/Source/Math/GPUMatrix.h index 6629297fb336..89339f4abe70 100644 --- a/Source/Math/GPUMatrix.h +++ b/Source/Math/GPUMatrix.h @@ -628,7 +628,7 @@ typedef GPUMatrix GPUSingleMatrix; template const char* CudaErrString(ERRTYPE x); // actual error function is defined inside .cu files template -static void CudaCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode) +static void CudaCall(ERRTYPE retCode, const char* exprString, const char* libName, ERRTYPE successCode, const char* msg="") { if (retCode != successCode) { @@ -643,7 +643,7 @@ static void CudaCall(ERRTYPE retCode, const char* exprString, const char* libNam #endif int currentCudaDevice; cudaGetDevice(¤tCudaDevice); - Microsoft::MSR::CNTK::RuntimeError("%s failure %d: %s ; GPU=%d ; hostname=%s ; expr=%s", libName, (int)retCode, CudaErrString(retCode), currentCudaDevice, hostname ? hostname : "?", exprString); + Microsoft::MSR::CNTK::RuntimeError("%s failure %d: %s ; GPU=%d ; hostname=%s ; expr=%s%s", libName, (int)retCode, CudaErrString(retCode), currentCudaDevice, hostname ? hostname : "?", exprString, msg); } catch (const std::exception& e) // catch, log, and rethrow since CUDA code sometimes hangs in destruction, so we'd never get to see the error { @@ -658,5 +658,6 @@ static void CudaCall(ERRTYPE retCode, const char* exprString, const char* libNam #define CUSPARSE_CALL(expr) (CudaCall((expr), #expr, "CUSPARSE", CUSPARSE_STATUS_SUCCESS)) #define CURAND_CALL(expr) (CudaCall((expr), #expr, "CURAND", CURAND_STATUS_SUCCESS)) #define CUDNN_CALL(expr) (CudaCall((expr), #expr, "cuDNN", CUDNN_STATUS_SUCCESS)) +#define CUDNN_CALL2(expr,m) (CudaCall((expr), #expr, "cuDNN", CUDNN_STATUS_SUCCESS, m)) #endif // CPUONLY diff --git a/Tests/UnitTests/V2LibraryTests/Image.h b/Tests/UnitTests/V2LibraryTests/Image.h index 4301c6942109..ff4a18b95aca 100644 --- a/Tests/UnitTests/V2LibraryTests/Image.h +++ b/Tests/UnitTests/V2LibraryTests/Image.h @@ -13,7 +13,7 @@ inline FunctionPtr ConvBNLayer(Variable input, size_t outFeatureMapCount, size_t auto scaleParams = Parameter({ NDShape::InferredDimension }, (float)scValue, device); auto runningMean = Constant({ NDShape::InferredDimension }, 0.0f, device); auto runningInvStd = Constant({ NDShape::InferredDimension }, 0.0f, device); - return BatchNormalization(convFunction, scaleParams, biasParams, runningMean, runningInvStd, spatial, (double)bnTimeConst, 0.0, 0.000000001 /* epsilon */); + return BatchNormalization(convFunction, scaleParams, biasParams, runningMean, runningInvStd, spatial, (double)bnTimeConst, 0.0, 1e-5 /* epsilon */); } inline FunctionPtr ConvBNReLULayer(Variable input, size_t outFeatureMapCount, size_t kernelWidth, size_t kernelHeight, size_t hStride, size_t vStride, double wScale, double bValue, double scValue, size_t bnTimeConst, bool spatial, const DeviceDescriptor& device) @@ -33,7 +33,7 @@ inline FunctionPtr ProjLayer(Variable wProj, Variable input, size_t hStride, siz size_t numInputChannels = input.Shape()[input.Shape().Rank() - 1]; auto c = Convolution(wProj, input, { hStride, vStride, numInputChannels }, { true }, { false }); - return BatchNormalization(c, sc, b, m, v, true /*spatial*/, (double)bnTimeConst); + return BatchNormalization(c, sc, b, m, v, true /*spatial*/, (double)bnTimeConst, 0, 1e-5, false); // TODO: cudnn engine does not work in Linux debug build here } inline FunctionPtr ResNetNode2(Variable input, size_t outFeatureMapCount, size_t kernelWidth, size_t kernelHeight, double wScale, double bValue, double scValue, size_t bnTimeConst, bool spatial, const DeviceDescriptor& device) diff --git a/Tutorials/CNTK_202_Language_Understanding.ipynb b/Tutorials/CNTK_202_Language_Understanding.ipynb index cc59c27ee876..aa9dc74f9947 100644 --- a/Tutorials/CNTK_202_Language_Understanding.ipynb +++ b/Tutorials/CNTK_202_Language_Understanding.ipynb @@ -495,7 +495,7 @@ " progress_printer = ProgressPrinter(tag='Evaluation')\n", "\n", " while True:\n", - " minibatch_size = 1000\n", + " minibatch_size = 500\n", " data = reader.next_minibatch(minibatch_size, input_map={ # fetch minibatch\n", " criterion.arguments[0]: reader.streams.query,\n", " criterion.arguments[1]: reader.streams.slot_labels\n", diff --git a/bindings/python/cntk/layers.py b/bindings/python/cntk/layers.py index 6bf2882fbe6f..f1d8733b3a89 100644 --- a/bindings/python/cntk/layers.py +++ b/bindings/python/cntk/layers.py @@ -271,7 +271,7 @@ def Dropout(prob): def BatchNormalization(map_rank=None, # if given then normalize only over this many dimensions. E.g. 1 to tie all (h,w) in a (C, H, W)-shaped input init_scale=1, normalization_time_constant=5000, blend_time_constant=0, - epsilon=0.00001, use_cntk_engine=True): + epsilon=0.00001, use_cntk_engine=False): # TODO: make map_rank a default option, once per-layer type defaults are implemented # parameters bound to this Function diff --git a/bindings/python/cntk/ops/tests/non_linear_test.py b/bindings/python/cntk/ops/tests/non_linear_test.py index 6f31e069da42..b18741a9bf1e 100644 --- a/bindings/python/cntk/ops/tests/non_linear_test.py +++ b/bindings/python/cntk/ops/tests/non_linear_test.py @@ -13,8 +13,11 @@ import numpy as np import pytest from .ops_test_utils import unittest_helper, _test_unary_op, _test_binary_op, AA, I, precision, PRECISION_TO_TYPE, cntk_device +from cntk.tests.test_utils import TOLERANCE_ABSOLUTE from cntk.utils import eval as cntk_eval, sanitize_dtype_cntk from .. import constant +from ..variables import Parameter, Constant +from cntk import set_default_device EPS_IN_LOG = 1e-37 # 1e-37 is the highest guaranteed precision # the backward result returned by CNTK log() for epsilon @@ -331,3 +334,39 @@ def test_op_hardmax(sample, device_id, precision): _test_unary_op(precision, device_id, hardmax, sample, expected_forward, expected_backward) + +@pytest.mark.parametrize("use_cudnn", [True, False]) +@pytest.mark.parametrize("sample", SAMPLES) +def test_op_batch_normalization(use_cudnn, sample, device_id, precision): + dtype = PRECISION_TO_TYPE[precision] + epsilon = 0.00001 + + t = AA(sample, dtype=dtype).reshape(-1,1,1) + mean = 1 + var = 2 + init_scale = 3 + init_bias = 4 + + forward = [(x - mean) / np.sqrt(var + epsilon) * init_scale + init_bias for x in t] + + expected_forward = AA(forward) + + scale = Parameter(init=AA([init_scale], dtype=dtype)) + bias = Parameter(init=AA([init_bias], dtype=dtype)) + run_mean = Constant(mean, shape=(1), dtype=dtype) + run_variance = Constant(var, shape=(1), dtype=dtype) + + from cntk import batch_normalization + + input = I(shape=(1), dtype=dtype, needs_gradient=False, name='input') + + op = batch_normalization(input, scale, bias, run_mean, run_variance, False, + epsilon=epsilon, + use_cudnn_engine=use_cudnn) + + forward_input = {input: t} + actual_forward = op.eval(forward_input) + + for res, exp in zip(actual_forward, expected_forward): + assert res.shape == AA(exp).shape + assert np.allclose(res, exp, atol=TOLERANCE_ABSOLUTE) \ No newline at end of file From ad587941f4400ad2943549bef3557e430877a095 Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Wed, 11 Jan 2017 10:15:46 +0000 Subject: [PATCH 092/120] Linux: Docker: fix build - CNTKCustomMKL is now on version 3 - CNTK build with OpenBlas was broken because of additional OpenMP dependencies in test binaries. --- Makefile | 4 ++-- Tools/docker/CNTK-CPUOnly-Image/Dockerfile | 4 ++-- Tools/docker/CNTK-GPU-1bit-Image/Dockerfile | 4 ++-- Tools/docker/CNTK-GPU-Image/Dockerfile | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 4eee8558de3a..f3ce11a8fcc3 100644 --- a/Makefile +++ b/Makefile @@ -1124,7 +1124,7 @@ $(UNITTEST_READER): $(UNITTEST_READER_OBJ) | $(HTKMLFREADER) $(HTKDESERIALIZERS) @echo $(SEPARATOR) @mkdir -p $(dir $@) @echo building $@ for $(ARCH) with build type $(BUILDTYPE) - $(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(BOOSTLIB_PATH)) $(patsubst %, $(RPATH)%, $(ORIGINLIBDIR) $(BOOSTLIB_PATH)) -o $@ $^ $(BOOSTLIBS) $(L_READER_LIBS) -ldl + $(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(BOOSTLIB_PATH)) $(patsubst %, $(RPATH)%, $(ORIGINLIBDIR) $(BOOSTLIB_PATH)) -o $@ $^ $(BOOSTLIBS) $(L_READER_LIBS) -ldl -fopenmp UNITTEST_NETWORK_SRC = \ $(SOURCEDIR)/../Tests/UnitTests/NetworkTests/AccumulatorNodeTests.cpp \ @@ -1221,7 +1221,7 @@ $(UNITTEST_BRAINSCRIPT): $(UNITTEST_BRAINSCRIPT_OBJ) | $(READER_LIBS) @echo $(SEPARATOR) @mkdir -p $(dir $@) @echo building $@ for $(ARCH) with build type $(BUILDTYPE) - $(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(GDK_NVML_LIB_PATH) $(BOOSTLIB_PATH)) $(patsubst %, $(RPATH)%, $(ORIGINLIBDIR) $(LIBPATH) $(BOOSTLIB_PATH)) -o $@ $^ $(BOOSTLIBS) $(LIBS) -ldl $(L_READER_LIBS) + $(CXX) $(LDFLAGS) $(patsubst %,-L%, $(LIBDIR) $(LIBPATH) $(GDK_NVML_LIB_PATH) $(BOOSTLIB_PATH)) $(patsubst %, $(RPATH)%, $(ORIGINLIBDIR) $(LIBPATH) $(BOOSTLIB_PATH)) -o $@ $^ $(BOOSTLIBS) $(LIBS) -ldl $(L_READER_LIBS) -fopenmp unittests: $(UNITTEST_EVAL) $(UNITTEST_READER) $(UNITTEST_NETWORK) $(UNITTEST_MATH) $(UNITTEST_BRAINSCRIPT) diff --git a/Tools/docker/CNTK-CPUOnly-Image/Dockerfile b/Tools/docker/CNTK-CPUOnly-Image/Dockerfile index cad11bfa982e..04a08f9954be 100644 --- a/Tools/docker/CNTK-CPUOnly-Image/Dockerfile +++ b/Tools/docker/CNTK-CPUOnly-Image/Dockerfile @@ -96,9 +96,9 @@ RUN PROTOBUF_VERSION=3.1.0 \ cd .. && \ rm -rf $PROTOBUF_STRING -# Install CNTK custom MKL, version 2 +# Install CNTK custom MKL, version 3 RUN mkdir /usr/local/CNTKCustomMKL && \ - wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-2.tgz | \ + wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-3.tgz | \ tar -xzf - -C /usr/local/CNTKCustomMKL # Install Kaldi diff --git a/Tools/docker/CNTK-GPU-1bit-Image/Dockerfile b/Tools/docker/CNTK-GPU-1bit-Image/Dockerfile index e58b99cd01b7..a576048dcc5b 100644 --- a/Tools/docker/CNTK-GPU-1bit-Image/Dockerfile +++ b/Tools/docker/CNTK-GPU-1bit-Image/Dockerfile @@ -111,9 +111,9 @@ RUN PROTOBUF_VERSION=3.1.0 \ cd .. && \ rm -rf $PROTOBUF_STRING -# Install CNTK custom MKL, version 2 +# Install CNTK custom MKL, version 3 RUN mkdir /usr/local/CNTKCustomMKL && \ - wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-2.tgz | \ + wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-3.tgz | \ tar -xzf - -C /usr/local/CNTKCustomMKL # Install Kaldi diff --git a/Tools/docker/CNTK-GPU-Image/Dockerfile b/Tools/docker/CNTK-GPU-Image/Dockerfile index b51329240a9e..4f15cda1e5c9 100644 --- a/Tools/docker/CNTK-GPU-Image/Dockerfile +++ b/Tools/docker/CNTK-GPU-Image/Dockerfile @@ -92,9 +92,9 @@ RUN PROTOBUF_VERSION=3.1.0 \ cd .. && \ rm -rf $PROTOBUF_STRING -# Install CNTK custom MKL, version 2 +# Install CNTK custom MKL, version 3 RUN mkdir /usr/local/CNTKCustomMKL && \ - wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-2.tgz | \ + wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-3.tgz | \ tar -xzf - -C /usr/local/CNTKCustomMKL # Install Kaldi From 2bd3cc8f22dd7c1681cb55b8d238f8182ac29169 Mon Sep 17 00:00:00 2001 From: Philipp Kranen Date: Wed, 4 Jan 2017 14:39:42 +0100 Subject: [PATCH 093/120] Feature extraction example using AlexNet from python --- .gitignore | 7 +- .../Image/DataSets/grocery/install_grocery.py | 27 ++++++ .../FeatureExtraction/FeatureExtraction.py | 88 +++++++++++++++++++ Examples/Image/FeatureExtraction/README.md | 21 +++++ Examples/Image/PretrainedModels/README.md | 3 + 5 files changed, 144 insertions(+), 2 deletions(-) create mode 100644 Examples/Image/DataSets/grocery/install_grocery.py create mode 100644 Examples/Image/FeatureExtraction/FeatureExtraction.py create mode 100644 Examples/Image/FeatureExtraction/README.md create mode 100644 Examples/Image/PretrainedModels/README.md diff --git a/.gitignore b/.gitignore index 2b17f305b713..e3073310c4a7 100644 --- a/.gitignore +++ b/.gitignore @@ -243,8 +243,11 @@ Examples/Image/DataSets/CIFAR-10/train_map.txt Examples/Image/DataSets/CIFAR-10/train_regrLabels.txt Examples/Image/DataSets/MNIST/Test-28x28_cntk_text.txt Examples/Image/DataSets/MNIST/Train-28x28_cntk_text.txt -Examples/Image/DataSets/grocery/ -Examples/Image/PretrainedModels/ +Examples/Image/DataSets/grocery/negative/ +Examples/Image/DataSets/grocery/positive/ +Examples/Image/DataSets/grocery/testImages/ +Examples/Image/DataSets/grocery/*.txt +Examples/Image/PretrainedModels/*.model Tutorials/HelloWorld-LogisticRegression/LR.txt.p Tutorials/HelloWorld-LogisticRegression/Models/ Tutorials/.ipynb_checkpoints/ diff --git a/Examples/Image/DataSets/grocery/install_grocery.py b/Examples/Image/DataSets/grocery/install_grocery.py new file mode 100644 index 000000000000..e2698250f8fc --- /dev/null +++ b/Examples/Image/DataSets/grocery/install_grocery.py @@ -0,0 +1,27 @@ +from __future__ import print_function +import os +import zipfile +import os +import os.path +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + +if __name__ == "__main__": + directory = "./../" + filename = directory + "/Grocery.zip" + if not os.path.exists(filename): + url = "https://www.cntk.ai/DataSets/Grocery/Grocery.zip" + print ('Downloading data from ' + url + '...') + urlretrieve(url, filename) + try: + print ('Extracting ' + filename + '...') + with zipfile.ZipFile(filename) as myzip: + myzip.extractall(directory) + finally: + os.remove(filename) + print ('Done.') + else: + print ('Data already available at ' + directory) + diff --git a/Examples/Image/FeatureExtraction/FeatureExtraction.py b/Examples/Image/FeatureExtraction/FeatureExtraction.py new file mode 100644 index 000000000000..59c3d91e39b0 --- /dev/null +++ b/Examples/Image/FeatureExtraction/FeatureExtraction.py @@ -0,0 +1,88 @@ +# Copyright (c) Microsoft. All rights reserved. + +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + +from __future__ import print_function +import os +import numpy as np +from cntk import load_model +from cntk.ops import combine +from cntk.io import MinibatchSource, ImageDeserializer, StreamDef, StreamDefs + +##################################################### +##################################################### +# helpers to print all node names +def dfs_walk(node, visited): + if node in visited: + return + visited.add(node) + print("visiting %s"%node.name) + if hasattr(node, 'root_function'): + node = node.root_function + for child in node.inputs: + dfs_walk(child, visited) + elif hasattr(node, 'is_output') and node.is_output: + dfs_walk(node.owner, visited) + +def print_all_node_names(model_file): + loaded_model = load_model(model_file) + dfs_walk(loaded_model, set()) +##################################################### +##################################################### + + +def create_mb_source(image_height, image_width, num_channels, map_file): + transforms = [ImageDeserializer.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear')] + return MinibatchSource(ImageDeserializer(map_file, StreamDefs( + features=StreamDef(field='image', transforms=transforms), # first column in map file is referred to as 'image' + labels=StreamDef(field='label', shape=1000)))) # and second as 'label'. TODO: add option to ignore labels + + +def eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects): + # load model and pick desired node as output + loaded_model = load_model(model_file) + node_in_graph = loaded_model.find_by_name(node_name) + output_nodes = combine([node_in_graph.owner]) + + # evaluate model and get desired node output + features_si = minibatch_source['features'] + with open(output_file, 'wb') as results_file: + for i in range(0, num_objects): + mb = minibatch_source.next_minibatch(1) + output = output_nodes.eval(mb[features_si]) + + # write results to file + out_values = output[0,0].flatten() + np.savetxt(results_file, out_values[np.newaxis], fmt="%.6f") + return + +if __name__ == '__main__': + # define location of model and data + base_folder = os.getcwd() + model_file = os.path.join(base_folder, "../PretrainedModels/AlexNetBS.model") + map_file = os.path.join(base_folder, "../DataSets/grocery/test.txt") + os.chdir(os.path.join(base_folder, "../DataSets/grocery/")) + + # create minibatch source + image_height = 227 + image_width = 227 + num_channels = 3 + minibatch_source = create_mb_source(image_height, image_width, num_channels, map_file) + + # use this to print all node names of the model (and knowledge of the model to pick the correct one) + # print_all_node_names(model_file) + + # use this to get 1000 class predictions (not yet softmaxed!) + # node_name = "z" + # output_file = os.path.join(base_folder, "predOutput.txt") + + # use this to get 4096 features from the last fc layer + node_name = "z.x._._" + output_file = os.path.join(base_folder, "fcOutput.txt") + + # evaluate model and write out the desired layer output + eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects=5) + + print("Done. Wrote output to %s" % output_file) diff --git a/Examples/Image/FeatureExtraction/README.md b/Examples/Image/FeatureExtraction/README.md new file mode 100644 index 000000000000..542f051ce88d --- /dev/null +++ b/Examples/Image/FeatureExtraction/README.md @@ -0,0 +1,21 @@ +# CNTK Examples: Image/FeatureExtraction + +## Overview + +|Data: |A small toy data set of food items in a fridge (grocery). +|:---------|:--- +|Purpose |Demonstrate how to evaluate and write out different layers of a trained model using python. +|Network |Pre-trained AlexNet model. +|Training |None, only evaluation of different layers of the model. + +## Running the example + +### Getting the data + +We use the `grocery` toy data set. To download it go to the folder [DataSets/grocery](../DataSets/grocery) and run `python install_grocery.py`. + +Additionally the example requires a pre-trained AlexNet model. Download this model from [https://www.cntk.ai/Models/AlexNet/AlexNetBS.model](https://www.cntk.ai/Models/AlexNet/AlexNetBS.model) and store it in [Examples/Image/PretrainedModels](../PretrainedModels). + +### Details + +Run `python FeatureExtraction.py` to generate the output of a specific layer. Please refer to the comments in the python code directly for how to choose different layers for evaluation. diff --git a/Examples/Image/PretrainedModels/README.md b/Examples/Image/PretrainedModels/README.md new file mode 100644 index 000000000000..ad902e363b2f --- /dev/null +++ b/Examples/Image/PretrainedModels/README.md @@ -0,0 +1,3 @@ +## Overview + +This folder holds pretrained models as required for different examples like [Examples/Image/Detection/FastRCNN](../Detection/FastRCNN) or [Examples/Image/FeatureExtraction](../FeatureExtraction). See the README.md files in the corresponding folder for details. \ No newline at end of file From 319386f528c7a0cc23fae4823f6c6def5fe198e6 Mon Sep 17 00:00:00 2001 From: Philipp Kranen Date: Thu, 5 Jan 2017 10:34:06 +0100 Subject: [PATCH 094/120] addressed CR comments --- .gitignore | 1 + .../Image/DataSets/grocery/install_grocery.py | 40 ++++++++----- .../Detection/FastRCNN/install_fastrcnn.py | 55 ++++++------------ .../FeatureExtraction/FeatureExtraction.py | 56 +++++++++---------- Examples/Image/FeatureExtraction/README.md | 8 +-- .../install_data_and_model.py | 21 +++++++ Examples/Image/PretrainedModels/README.md | 2 +- .../Image/PretrainedModels/models_util.py | 37 ++++++++++++ Tools/samples.json | 8 +++ 9 files changed, 139 insertions(+), 89 deletions(-) create mode 100644 Examples/Image/FeatureExtraction/install_data_and_model.py create mode 100644 Examples/Image/PretrainedModels/models_util.py diff --git a/.gitignore b/.gitignore index e3073310c4a7..51a3ea943fe1 100644 --- a/.gitignore +++ b/.gitignore @@ -248,6 +248,7 @@ Examples/Image/DataSets/grocery/positive/ Examples/Image/DataSets/grocery/testImages/ Examples/Image/DataSets/grocery/*.txt Examples/Image/PretrainedModels/*.model +Examples/Image/FeatureExtraction/*.txt Tutorials/HelloWorld-LogisticRegression/LR.txt.p Tutorials/HelloWorld-LogisticRegression/Models/ Tutorials/.ipynb_checkpoints/ diff --git a/Examples/Image/DataSets/grocery/install_grocery.py b/Examples/Image/DataSets/grocery/install_grocery.py index e2698250f8fc..51380473d209 100644 --- a/Examples/Image/DataSets/grocery/install_grocery.py +++ b/Examples/Image/DataSets/grocery/install_grocery.py @@ -1,27 +1,37 @@ +# Copyright (c) Microsoft. All rights reserved. + +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + from __future__ import print_function -import os import zipfile import os -import os.path -try: +try: from urllib.request import urlretrieve except ImportError: from urllib import urlretrieve -if __name__ == "__main__": - directory = "./../" - filename = directory + "/Grocery.zip" - if not os.path.exists(filename): - url = "https://www.cntk.ai/DataSets/Grocery/Grocery.zip" - print ('Downloading data from ' + url + '...') - urlretrieve(url, filename) +def download_grocery_data(): + base_folder = os.path.dirname(os.path.abspath(__file__)) + dataset_folder = os.path.join(base_folder, "..") + if not os.path.exists(os.path.join(dataset_folder, "grocery", "testImages")): + filename = os.path.join(dataset_folder, "Grocery.zip") + if not os.path.exists(filename): + url = "https://www.cntk.ai/DataSets/Grocery/Grocery.zip" + print('Downloading data from ' + url + '...') + urlretrieve(url, filename) + try: - print ('Extracting ' + filename + '...') + print('Extracting ' + filename + '...') with zipfile.ZipFile(filename) as myzip: - myzip.extractall(directory) + myzip.extractall(dataset_folder) finally: os.remove(filename) - print ('Done.') + print('Done.') else: - print ('Data already available at ' + directory) - + print('Data already available at ' + dataset_folder + '/grocery') + +if __name__ == "__main__": + download_grocery_data() + \ No newline at end of file diff --git a/Examples/Image/Detection/FastRCNN/install_fastrcnn.py b/Examples/Image/Detection/FastRCNN/install_fastrcnn.py index 668bf40ef77a..319832499922 100644 --- a/Examples/Image/Detection/FastRCNN/install_fastrcnn.py +++ b/Examples/Image/Detection/FastRCNN/install_fastrcnn.py @@ -1,40 +1,19 @@ +# Copyright (c) Microsoft. All rights reserved. + +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + from __future__ import print_function -import os import zipfile -import os -import os.path -try: - from urllib.request import urlretrieve -except ImportError: - from urllib import urlretrieve - -if __name__ == "__main__": - directory = "./../../DataSets" - if not os.path.exists(directory + "/Grocery"): - filename = directory + "/Grocery.zip" - if not os.path.exists(filename): - url = "https://www.cntk.ai/DataSets/Grocery/Grocery.zip" - print ('Downloading data from ' + url + '...') - urlretrieve(url, filename) - try: - print ('Extracting ' + filename + '...') - with zipfile.ZipFile(filename) as myzip: - myzip.extractall(directory) - finally: - os.remove(filename) - print ('Done.') - else: - print ('Data already available at ' + directory) - - directory = "./../../PretrainedModels" - if not os.path.exists(directory): - os.makedirs(directory) - filename = directory + "/AlexNet.model" - if not os.path.exists(filename): - url = "https://www.cntk.ai/Models/AlexNet/AlexNet.model" - print ('Downloading model from ' + url + ', may take a while...') - urlretrieve(url, filename) - print ('Saved model as ' + filename) - else: - print ('CNTK model already available at ' + filename) - +import os, sys + +base_folder = os.path.dirname(os.path.abspath(__file__)) + +sys.path.append(os.path.join(base_folder, "..", "..", "DataSets", "grocery")) +from install_grocery import download_grocery_data +download_grocery_data() + +sys.path.append(os.path.join(base_folder, "..", "..", "PretrainedModels")) +from models_util import download_model_by_name +download_model_by_name("AlexNet") diff --git a/Examples/Image/FeatureExtraction/FeatureExtraction.py b/Examples/Image/FeatureExtraction/FeatureExtraction.py index 59c3d91e39b0..9990922d936a 100644 --- a/Examples/Image/FeatureExtraction/FeatureExtraction.py +++ b/Examples/Image/FeatureExtraction/FeatureExtraction.py @@ -7,37 +7,29 @@ from __future__ import print_function import os import numpy as np -from cntk import load_model +from cntk import load_model, graph from cntk.ops import combine from cntk.io import MinibatchSource, ImageDeserializer, StreamDef, StreamDefs +from cntk import graph -##################################################### -##################################################### -# helpers to print all node names -def dfs_walk(node, visited): - if node in visited: - return - visited.add(node) - print("visiting %s"%node.name) - if hasattr(node, 'root_function'): - node = node.root_function - for child in node.inputs: - dfs_walk(child, visited) - elif hasattr(node, 'is_output') and node.is_output: - dfs_walk(node.owner, visited) - -def print_all_node_names(model_file): +# helper to print all node names +def print_all_node_names(model_file, is_BrainScript): loaded_model = load_model(model_file) - dfs_walk(loaded_model, set()) -##################################################### -##################################################### + if is_BrainScript: + loaded_model = combine([loaded_model.outputs[0]]) + node_list = graph.depth_first_search(loaded_model, lambda x: x.is_output) + print("printing node information in the format") + print("node name (tensor shape)") + for node in node_list: + print(node.name, node.shape) def create_mb_source(image_height, image_width, num_channels, map_file): transforms = [ImageDeserializer.scale(width=image_width, height=image_height, channels=num_channels, interpolations='linear')] return MinibatchSource(ImageDeserializer(map_file, StreamDefs( features=StreamDef(field='image', transforms=transforms), # first column in map file is referred to as 'image' - labels=StreamDef(field='label', shape=1000)))) # and second as 'label'. TODO: add option to ignore labels + labels=StreamDef(field='label', shape=1000))), # and second as 'label'. TODO: add option to ignore labels + randomize=False) def eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects): @@ -56,14 +48,16 @@ def eval_and_write(model_file, node_name, output_file, minibatch_source, num_obj # write results to file out_values = output[0,0].flatten() np.savetxt(results_file, out_values[np.newaxis], fmt="%.6f") - return if __name__ == '__main__': - # define location of model and data - base_folder = os.getcwd() - model_file = os.path.join(base_folder, "../PretrainedModels/AlexNetBS.model") - map_file = os.path.join(base_folder, "../DataSets/grocery/test.txt") - os.chdir(os.path.join(base_folder, "../DataSets/grocery/")) + # define location of model and data and check existence + base_folder = os.path.dirname(os.path.abspath(__file__)) + model_file = os.path.join(base_folder, "..", "PretrainedModels", "ResNet_18.model") + map_file = os.path.join(base_folder, "..", "DataSets", "grocery", "test.txt") + os.chdir(os.path.join(base_folder, "..", "DataSets", "grocery")) + if not (os.path.exists(model_file) and os.path.exists(map_file)): + print("Please run 'python install_data_and_model.py' first to get the required data and model.") + exit(0) # create minibatch source image_height = 227 @@ -72,15 +66,15 @@ def eval_and_write(model_file, node_name, output_file, minibatch_source, num_obj minibatch_source = create_mb_source(image_height, image_width, num_channels, map_file) # use this to print all node names of the model (and knowledge of the model to pick the correct one) - # print_all_node_names(model_file) + print_all_node_names(model_file, True) # use this to get 1000 class predictions (not yet softmaxed!) # node_name = "z" # output_file = os.path.join(base_folder, "predOutput.txt") - # use this to get 4096 features from the last fc layer - node_name = "z.x._._" - output_file = os.path.join(base_folder, "fcOutput.txt") + # use this to get 512 features from the last but one layer of ResNet_18 + node_name = "z.x" + output_file = os.path.join(base_folder, "layerOutput.txt") # evaluate model and write out the desired layer output eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects=5) diff --git a/Examples/Image/FeatureExtraction/README.md b/Examples/Image/FeatureExtraction/README.md index 542f051ce88d..9670754edf55 100644 --- a/Examples/Image/FeatureExtraction/README.md +++ b/Examples/Image/FeatureExtraction/README.md @@ -4,7 +4,7 @@ |Data: |A small toy data set of food items in a fridge (grocery). |:---------|:--- -|Purpose |Demonstrate how to evaluate and write out different layers of a trained model using python. +|Purpose |Demonstrate how to evaluate and write out different layers of a trained model using Python. |Network |Pre-trained AlexNet model. |Training |None, only evaluation of different layers of the model. @@ -12,10 +12,10 @@ ### Getting the data -We use the `grocery` toy data set. To download it go to the folder [DataSets/grocery](../DataSets/grocery) and run `python install_grocery.py`. +We use the `grocery` toy data set ([Examples/Image/DataSets/grocery](../DataSets/grocery)) and a pre-trained AlexNet model [Examples/Image/PretrainedModels/AlexNetBS.model](../PretrainedModels). To download both run -Additionally the example requires a pre-trained AlexNet model. Download this model from [https://www.cntk.ai/Models/AlexNet/AlexNetBS.model](https://www.cntk.ai/Models/AlexNet/AlexNetBS.model) and store it in [Examples/Image/PretrainedModels](../PretrainedModels). +`python install_data_and_model.py` ### Details -Run `python FeatureExtraction.py` to generate the output of a specific layer. Please refer to the comments in the python code directly for how to choose different layers for evaluation. +Run `python FeatureExtraction.py` to generate the output of a specific layer. Please refer to the comments in the Python code directly for how to choose different layers for evaluation. diff --git a/Examples/Image/FeatureExtraction/install_data_and_model.py b/Examples/Image/FeatureExtraction/install_data_and_model.py new file mode 100644 index 000000000000..7fffd997a02d --- /dev/null +++ b/Examples/Image/FeatureExtraction/install_data_and_model.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft. All rights reserved. + +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + +from __future__ import print_function +import zipfile +import os, sys +import os.path + +base_folder = os.path.dirname(os.path.abspath(__file__)) + +sys.path.append(os.path.join(base_folder, "..", "DataSets", "grocery")) +from install_grocery import download_grocery_data +download_grocery_data() + +sys.path.append(os.path.join(base_folder, "..", "PretrainedModels")) +from models_util import download_model_by_name +download_model_by_name("ResNet_18") + diff --git a/Examples/Image/PretrainedModels/README.md b/Examples/Image/PretrainedModels/README.md index ad902e363b2f..58ecb4c9d3bc 100644 --- a/Examples/Image/PretrainedModels/README.md +++ b/Examples/Image/PretrainedModels/README.md @@ -1,3 +1,3 @@ ## Overview -This folder holds pretrained models as required for different examples like [Examples/Image/Detection/FastRCNN](../Detection/FastRCNN) or [Examples/Image/FeatureExtraction](../FeatureExtraction). See the README.md files in the corresponding folder for details. \ No newline at end of file +This folder holds pretrained models as required for different examples like [Examples/Image/Detection/FastRCNN](../Detection/FastRCNN) or [Examples/Image/FeatureExtraction](../FeatureExtraction). See the README.md file in the corresponding folder for details. \ No newline at end of file diff --git a/Examples/Image/PretrainedModels/models_util.py b/Examples/Image/PretrainedModels/models_util.py new file mode 100644 index 000000000000..600b671e19ca --- /dev/null +++ b/Examples/Image/PretrainedModels/models_util.py @@ -0,0 +1,37 @@ +# Copyright (c) Microsoft. All rights reserved. + +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + +from __future__ import print_function +import os +try: + from urllib.request import urlretrieve +except ImportError: + from urllib import urlretrieve + +def download_model(model_file_name, model_url): + model_dir = os.path.dirname(os.path.abspath(__file__)) + filename = os.path.join(model_dir, model_file_name) + if not os.path.exists(filename): + print('Downloading model from ' + model_url + ', may take a while...') + urlretrieve(model_url, filename) + print('Saved model as ' + filename) + else: + print('CNTK model already available at ' + filename) + +def download_model_by_name(model_name): + if not model_name.endswith('.model'): + model_name = model_name + '.model' + + modelNameToUrl = { + 'AlexNet.model': 'https://www.cntk.ai/Models/AlexNet/AlexNet.model', + 'AlexNetBS.model': 'https://www.cntk.ai/Models/AlexNet/AlexNetBS.model', + 'ResNet_18.model': 'https://www.cntk.ai/Models/ResNet/ResNet_18.model', + } + + if not model_name in modelNameToUrl: + print("ERROR: Unknown model name '%s'." % model_name) + else: + download_model(model_name, modelNameToUrl[model_name]) diff --git a/Tools/samples.json b/Tools/samples.json index 6ccb994bfd5d..ff85203453b9 100644 --- a/Tools/samples.json +++ b/Tools/samples.json @@ -23,6 +23,14 @@ "language": ["C++"], "type": ["Example"] }, + { + "category": ["Image"], + "name": "Feature Extraction", + "url": "https://github.com/Microsoft/CNTK/tree/master/Examples/Image/FeatureExtraction", + "description": "Evaluate and write out different layers of a trained model using Python.", + "language": ["Python"], + "type": ["Recipe"] + }, { "category": ["Image"], "name": "Video Action Recognition", From fc7331ae6f56fa1d6651d13a70859ce45b05a575 Mon Sep 17 00:00:00 2001 From: Philipp Kranen Date: Tue, 10 Jan 2017 18:13:38 +0100 Subject: [PATCH 095/120] added test for feature extraction example --- .gitignore | 1 + .../FeatureExtraction/FeatureExtraction.py | 10 ++-- .../feature_extraction_expected_output.txt | 2 + .../Examples/feature_extraction_test.py | 50 +++++++++++++++++++ 4 files changed, 59 insertions(+), 4 deletions(-) create mode 100644 Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_expected_output.txt create mode 100644 Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py diff --git a/.gitignore b/.gitignore index 51a3ea943fe1..1aeed9beea98 100644 --- a/.gitignore +++ b/.gitignore @@ -249,6 +249,7 @@ Examples/Image/DataSets/grocery/testImages/ Examples/Image/DataSets/grocery/*.txt Examples/Image/PretrainedModels/*.model Examples/Image/FeatureExtraction/*.txt +Tests/EndToEndTests/CNTKv2Python/Examples/layerOutput.txt Tutorials/HelloWorld-LogisticRegression/LR.txt.p Tutorials/HelloWorld-LogisticRegression/Models/ Tutorials/.ipynb_checkpoints/ diff --git a/Examples/Image/FeatureExtraction/FeatureExtraction.py b/Examples/Image/FeatureExtraction/FeatureExtraction.py index 9990922d936a..f7f4ac6bb7cd 100644 --- a/Examples/Image/FeatureExtraction/FeatureExtraction.py +++ b/Examples/Image/FeatureExtraction/FeatureExtraction.py @@ -12,8 +12,9 @@ from cntk.io import MinibatchSource, ImageDeserializer, StreamDef, StreamDefs from cntk import graph + # helper to print all node names -def print_all_node_names(model_file, is_BrainScript): +def print_all_node_names(model_file, is_BrainScript=True): loaded_model = load_model(model_file) if is_BrainScript: loaded_model = combine([loaded_model.outputs[0]]) @@ -39,6 +40,7 @@ def eval_and_write(model_file, node_name, output_file, minibatch_source, num_obj output_nodes = combine([node_in_graph.owner]) # evaluate model and get desired node output + print("Evaluating model for output node %s" % node_name) features_si = minibatch_source['features'] with open(output_file, 'wb') as results_file: for i in range(0, num_objects): @@ -60,13 +62,13 @@ def eval_and_write(model_file, node_name, output_file, minibatch_source, num_obj exit(0) # create minibatch source - image_height = 227 - image_width = 227 + image_height = 224 + image_width = 224 num_channels = 3 minibatch_source = create_mb_source(image_height, image_width, num_channels, map_file) # use this to print all node names of the model (and knowledge of the model to pick the correct one) - print_all_node_names(model_file, True) + # print_all_node_names(model_file) # use this to get 1000 class predictions (not yet softmaxed!) # node_name = "z" diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_expected_output.txt b/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_expected_output.txt new file mode 100644 index 000000000000..5c426a6c9f0e --- /dev/null +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_expected_output.txt @@ -0,0 +1,2 @@ +2.189164 0.448430 0.596691 2.670019 0.188035 0.000000 0.503985 0.866248 0.000000 1.060764 0.014179 0.671274 0.011275 1.593267 2.336892 1.338118 0.091209 0.161422 2.044161 0.328887 0.764157 0.849308 0.000000 2.972178 1.877296 3.378828 0.147710 0.082341 0.000000 1.426285 1.171324 0.000000 0.587745 1.116722 0.260962 0.975826 0.253545 0.833149 0.004671 3.650815 2.243172 2.981478 1.369477 1.856299 0.000000 0.591245 1.436951 5.092325 2.669012 0.713226 0.167314 0.669807 0.241324 5.666718 0.185367 0.311784 1.289877 0.848081 0.061453 0.180671 1.016511 0.239284 0.000000 0.524049 0.053789 1.157341 0.733647 0.176215 0.000000 0.552459 0.080060 6.720785 3.992243 0.000000 1.492989 0.059612 0.109433 0.599156 0.372988 1.654256 0.386433 1.175062 0.904472 0.233078 3.586270 0.000000 1.191918 0.054731 1.223395 0.133801 2.029352 0.592752 0.512741 0.492804 1.749217 1.077109 0.150474 1.318103 2.569207 0.400528 0.900308 1.119933 0.000000 0.399872 0.065316 0.244920 1.425678 0.000000 0.000000 0.014928 0.240836 0.195303 1.610985 0.475520 0.009774 1.117716 0.147407 2.322429 0.374848 0.227184 2.781121 0.340302 0.000000 1.644084 0.580358 0.146195 0.044728 0.000000 0.047906 1.599745 0.000000 0.000000 0.564236 1.034859 2.078934 0.971860 0.842374 0.456216 4.362343 0.027419 0.100437 1.487627 0.000000 0.079374 0.151621 4.859155 1.163515 0.045508 0.726114 0.000000 0.441255 0.000000 1.049023 1.190301 0.357938 0.514367 0.108224 0.000000 7.275521 0.000000 2.376608 1.696667 0.503640 0.150401 0.362030 0.000000 2.375431 2.608376 0.054331 0.078532 0.000000 3.267272 0.000000 0.451466 0.236907 0.278920 0.662506 1.067163 0.321966 0.992632 0.568973 1.227151 0.934294 4.084160 2.255595 0.035475 2.736990 0.005630 0.008481 0.564598 0.655451 1.021853 0.044663 1.279812 0.130560 0.123734 5.721457 0.089845 2.894609 0.583208 0.012870 0.000000 0.000000 0.422126 0.024934 3.958275 3.434095 0.263562 0.884978 0.308682 0.959709 0.151003 0.668906 0.410180 2.744696 0.258870 0.694474 0.539015 0.000000 0.534852 1.616657 0.187229 2.242972 0.292112 0.478580 1.018166 0.000000 1.174273 0.042462 0.697203 0.015845 0.428079 0.207730 2.386863 0.305964 0.603232 0.549219 0.071632 1.632386 0.804260 1.055701 0.428177 1.237638 0.311789 0.000000 0.119419 1.757029 0.016615 0.053262 0.677162 0.566289 2.048805 0.373024 0.945374 0.471650 7.708382 1.050909 0.004225 0.217826 0.958933 0.000000 0.379405 1.863586 1.203233 0.703975 0.811652 0.831063 0.000000 0.289609 1.265273 1.143159 0.083386 0.471813 0.366351 1.936840 2.191843 0.445009 0.336688 1.615357 0.322373 0.306104 1.500753 1.334141 0.074543 0.303082 2.846482 0.478705 0.088877 0.000000 0.246166 1.297087 0.064442 0.428281 0.158808 0.298951 0.905872 0.309868 0.017328 0.222334 0.013264 0.758645 1.347791 2.107569 0.329759 3.707090 0.787247 2.767848 1.050160 0.794756 3.160860 2.628176 1.141201 0.364364 1.418092 1.756548 0.044056 0.610406 0.054066 0.045545 0.138234 0.859035 2.727859 0.271020 0.248005 0.000000 0.171741 0.017387 0.248281 0.000000 3.458487 1.692802 4.731974 0.228206 0.038264 0.000000 2.337620 1.094873 0.000000 0.277675 0.951723 0.299119 1.877216 0.179941 1.272291 0.591268 0.126608 0.000000 0.873241 0.000000 0.000000 0.266069 0.297507 0.720554 0.000000 0.615150 1.042867 0.724737 0.864095 0.000000 0.086834 0.028971 2.779119 1.309942 0.645173 0.063998 1.035733 8.227171 0.300342 0.168868 0.922444 0.042879 0.454497 0.749772 0.131785 4.500103 2.819220 0.402055 0.064713 0.350529 3.085061 0.245873 5.786015 0.500738 0.000000 0.451133 0.056721 0.625805 0.421861 0.125893 0.000000 0.049973 0.467355 1.474214 0.230333 0.205526 0.000000 0.588911 21.126709 0.000261 0.772254 1.015508 0.010382 0.605906 0.012221 0.372078 0.593083 2.318626 2.433967 3.658823 0.091459 0.752087 0.303049 1.335710 0.951884 1.587140 2.612931 2.356208 0.035478 0.404046 0.214384 0.869077 0.203426 0.885217 0.121133 0.083401 0.872889 0.450872 1.479761 0.692724 0.473729 0.000000 1.389408 0.578537 0.149265 0.136507 0.000000 0.352463 1.462003 3.707973 0.006354 0.352618 0.065227 1.113809 0.000000 0.134100 1.013990 0.118864 0.757547 3.353299 0.494120 0.000000 1.898655 0.042222 1.284334 1.216094 0.074351 2.361221 1.715659 0.476350 0.036986 0.044048 1.530126 0.452030 0.000000 0.000000 0.540503 0.023323 2.559479 0.084827 0.051701 0.485452 1.142867 0.000000 0.086304 2.123181 2.598170 0.000000 0.336874 1.318214 1.586256 1.764792 0.038910 0.000000 0.285535 1.203824 1.632958 1.390715 0.030092 1.706043 0.495937 1.226338 0.017673 0.001241 0.048798 0.137762 0.840458 0.777166 3.069610 2.325499 0.000000 0.658468 1.027870 2.933879 0.311581 0.075263 1.284806 0.044623 0.572469 2.202977 1.203644 5.463371 2.070741 +0.000000 0.017158 1.308856 2.566151 0.371068 0.103814 0.000000 0.000000 0.276682 0.103976 0.218594 0.294717 0.068471 2.626514 3.420052 2.071716 0.035916 1.886685 0.196562 2.894245 0.141155 0.340184 0.459653 1.151823 3.351244 2.168559 0.991209 2.527645 0.000000 0.385926 0.159789 0.097411 1.440266 0.000000 0.066173 0.069022 0.059697 3.851507 0.191514 4.195241 0.947162 0.163818 0.363006 0.494454 0.370198 0.000000 0.051315 0.114526 2.119674 0.918644 0.000000 1.914023 0.059590 0.241850 0.105333 0.040012 4.614347 0.075857 1.682172 0.461151 2.439267 0.029704 1.107474 2.399470 0.447221 0.215349 0.000000 0.604423 0.209529 1.115550 1.068401 3.769885 1.351840 0.000000 0.935095 0.000000 0.169433 0.981671 0.000000 0.630210 0.147057 0.005509 0.000000 0.215702 0.125837 0.000000 1.316285 1.069647 0.246724 0.740145 0.385063 4.241658 0.045264 0.000000 0.000000 0.660849 0.387990 0.852547 0.320606 0.416138 0.729604 2.984382 0.057847 5.593803 0.492670 0.000647 0.364786 0.238994 1.385028 0.457113 0.399336 1.179515 0.088019 0.906877 0.213906 3.835125 0.204377 0.948530 2.615331 0.246635 0.030776 0.468028 0.009395 0.000000 1.233382 0.000000 0.010915 0.264430 0.491545 0.202741 0.243347 0.534565 0.204833 1.778670 0.041599 0.311156 0.000000 0.651703 0.353104 1.058847 0.940752 1.696938 0.783776 0.000000 0.579922 3.201265 0.855206 0.042445 0.000000 0.540176 1.335626 0.000000 4.956754 1.866077 0.014960 0.647758 0.000000 0.016525 3.626026 0.162796 2.431448 0.561922 2.938876 0.745134 0.000000 0.795261 0.000000 0.451212 1.083240 0.064236 1.036051 3.972299 3.740246 0.185833 0.311948 3.125623 1.614736 0.925928 0.057742 0.000000 1.001111 1.077264 0.160579 1.173602 0.014741 0.856172 0.924279 0.002767 0.004766 0.247878 0.107515 0.027091 0.001270 4.925105 0.014847 0.000000 0.244307 0.126039 2.917488 0.119547 0.087015 1.955483 0.000000 1.801343 0.274076 0.000000 0.559350 2.953540 2.355929 0.053902 0.128179 0.070026 0.139349 0.796951 0.118379 0.209053 0.047648 0.381132 0.050120 0.000000 0.009329 0.000000 0.045440 0.331172 0.250197 0.765427 0.046808 1.360799 0.101734 2.412263 0.000000 0.211911 5.006120 0.109439 0.000000 1.183158 6.273687 0.192492 3.023909 0.122360 1.341695 2.797058 3.705102 4.513747 0.713611 0.471687 1.199300 0.000000 0.007854 0.000000 0.374159 2.717425 0.026859 0.388711 2.455977 7.636169 0.247321 0.021130 2.597980 0.873560 0.416454 0.341172 0.000000 0.034929 1.035370 2.508342 0.070925 0.031007 0.533384 1.822999 2.494639 0.000000 1.203350 3.402665 0.376142 1.351579 0.942680 1.789959 0.788242 0.480232 3.805908 4.433710 1.477929 0.020494 1.105555 3.879987 0.577592 0.081976 0.109887 0.309741 0.274980 0.000000 0.040150 0.000000 0.011536 1.820660 1.986709 0.907040 0.000000 0.115869 3.026625 2.313065 0.018158 1.381130 2.301114 2.774600 0.138009 8.898828 1.939900 0.500612 0.533512 0.077774 1.889544 0.000000 0.000000 0.000000 0.000000 0.037285 0.042392 1.012028 0.000000 2.187261 0.437690 2.120923 0.000000 0.007609 0.308478 0.150089 0.000000 5.832235 0.657925 0.673686 0.021408 5.001243 0.397465 1.044956 0.717309 0.000000 0.076221 0.427965 0.460918 3.797780 0.000000 0.469655 0.021370 0.546371 0.562759 0.000472 0.000000 0.000000 0.668854 1.880855 2.951498 0.000000 2.700269 1.070793 0.968839 0.302267 0.135049 0.377998 0.055368 5.257760 1.137352 0.264256 0.409590 0.849936 1.049555 2.074064 0.168978 0.109963 0.569531 0.850015 1.686424 0.017175 0.000000 0.358730 0.772511 0.012291 0.990480 0.000000 2.113945 2.300197 0.720571 0.130953 1.179834 0.583624 0.000000 1.089285 1.140347 0.018167 0.014961 0.000000 0.160427 0.664246 1.396044 0.000000 0.000000 12.496775 0.000000 0.639026 0.009751 0.865042 0.000000 2.009616 0.158041 0.217022 0.007934 0.949908 0.007012 3.448552 0.000000 0.352707 0.000000 0.203183 0.000000 0.513840 5.319386 1.306992 1.035419 0.288410 0.781021 1.108952 2.676233 3.745963 0.000000 1.491164 0.009149 1.419904 0.284951 0.004064 0.000000 0.263136 1.427216 0.161527 0.026253 0.573844 0.646333 0.228197 0.199646 0.033382 0.003021 0.000000 2.881136 0.000000 0.201713 1.657715 1.725987 0.161795 0.412296 0.940082 0.451645 0.842232 0.241130 5.502472 0.220431 3.468875 0.122373 8.885290 0.000000 0.000000 0.101310 1.855898 0.426680 0.633068 0.000000 0.829681 0.255880 3.555860 0.549875 0.217912 0.904038 1.257358 0.000000 0.049522 0.228277 0.733695 0.396478 0.000000 0.160548 0.000000 0.251291 0.066225 0.414977 0.752671 0.085081 0.081291 0.428759 0.000000 0.802672 0.483281 0.358478 0.077576 0.129063 0.029697 0.181557 0.000000 0.001637 0.303331 3.469016 1.069852 3.350169 0.416463 1.920919 1.266463 0.329184 1.361397 0.000000 0.702556 0.000000 0.015971 5.714453 2.374993 diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py b/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py new file mode 100644 index 000000000000..6794c8646947 --- /dev/null +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py @@ -0,0 +1,50 @@ +# Copyright (c) Microsoft. All rights reserved. + +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + +import numpy as np +import os +import sys +from cntk.ops.tests.ops_test_utils import cntk_device +from cntk.cntk_py import DeviceKind_GPU +from cntk.device import set_default_device +import pytest + +abs_path = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(abs_path, "..", "..", "..", "..", "Examples", "Image", "FeatureExtraction")) +from FeatureExtraction import create_mb_source, eval_and_write + +TOLERANCE_ABSOLUTE = 2E-2 + +def test_feature_extraction(device_id): + if cntk_device(device_id).type() != DeviceKind_GPU: + pytest.skip('test only runs on GPU') # due to batch normalization in ResNet_18 + set_default_device(cntk_device(device_id)) + + base_path = os.path.dirname(os.path.abspath(__file__)) + externalData = 'cCNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ + if externalData: + extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] + print("Reading data and model from %s" % extPath) + model_file = os.path.join(extPath, *"PreTrainedModels/ResNet/v1/ResNet_18.model".split("/")) + map_file = os.path.join(extPath, *"Image/CIFAR/v0/cifar-10-batches-py/test_map.txt".split("/")) + else: + model_file = os.path.join(base_path, *"../../../../Examples/Image/PretrainedModels/ResNet_18.model".split("/")) + map_file = os.path.join(base_path, *"../../../../Examples/Image/DataSets/CIFAR-10/test_map.txt".split("/")) + + os.chdir(os.path.join(os.path.dirname(map_file), '..')) + + minibatch_source = create_mb_source(224, 224, 3, map_file) + node_name = "z.x" + output_file = os.path.join(base_path, "layerOutput.txt") + eval_and_write(model_file, node_name, output_file, minibatch_source, num_objects=2) + + expected_output_file = os.path.join(base_path, "feature_extraction_expected_output.txt") + output = np.fromfile(output_file) + expected_output = np.fromfile(expected_output_file) + + print(output.shape) + print(expected_output.shape) + assert np.allclose(output, expected_output, atol=TOLERANCE_ABSOLUTE) From a4e94eb1ff38e6a48bf31cbdfad3179e96436b58 Mon Sep 17 00:00:00 2001 From: Philipp Kranen Date: Tue, 10 Jan 2017 18:19:38 +0100 Subject: [PATCH 096/120] Moving feature extraction example to ResNet_18 --- Examples/Image/FeatureExtraction/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Examples/Image/FeatureExtraction/README.md b/Examples/Image/FeatureExtraction/README.md index 9670754edf55..460bf5674950 100644 --- a/Examples/Image/FeatureExtraction/README.md +++ b/Examples/Image/FeatureExtraction/README.md @@ -5,14 +5,14 @@ |Data: |A small toy data set of food items in a fridge (grocery). |:---------|:--- |Purpose |Demonstrate how to evaluate and write out different layers of a trained model using Python. -|Network |Pre-trained AlexNet model. +|Network |Pre-trained ResNet_18 model. |Training |None, only evaluation of different layers of the model. ## Running the example ### Getting the data -We use the `grocery` toy data set ([Examples/Image/DataSets/grocery](../DataSets/grocery)) and a pre-trained AlexNet model [Examples/Image/PretrainedModels/AlexNetBS.model](../PretrainedModels). To download both run +We use the `grocery` toy data set ([Examples/Image/DataSets/grocery](../DataSets/grocery)) and a pre-trained ResNet_18 model [Examples/Image/PretrainedModels/ResNet_18.model](../PretrainedModels). To download both run `python install_data_and_model.py` From 18f35d7da6168d67384884c56573ab8a0450f852 Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Wed, 11 Jan 2017 13:22:47 +0000 Subject: [PATCH 097/120] Address CR comments --- Makefile | 2 +- Tools/docker/CNTK-CPUOnly-Image/Dockerfile | 7 ++++--- Tools/docker/CNTK-GPU-1bit-Image/Dockerfile | 7 ++++--- Tools/docker/CNTK-GPU-Image/Dockerfile | 7 ++++--- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index f3ce11a8fcc3..2b274daffe79 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ # defaults to release # MKL_PATH= path to CNTK custom MKL installation # only needed if MATHLIB=mkl -# CNTK_CUSTOM_MKL_VERSION=2 +# CNTK_CUSTOM_MKL_VERSION=3 # version for the CNTK custom MKL installation # MKL_THREADING=parallel|sequential # only needed if MATHLIB=mkl diff --git a/Tools/docker/CNTK-CPUOnly-Image/Dockerfile b/Tools/docker/CNTK-CPUOnly-Image/Dockerfile index 04a08f9954be..777ba1e7efcb 100644 --- a/Tools/docker/CNTK-CPUOnly-Image/Dockerfile +++ b/Tools/docker/CNTK-CPUOnly-Image/Dockerfile @@ -96,9 +96,10 @@ RUN PROTOBUF_VERSION=3.1.0 \ cd .. && \ rm -rf $PROTOBUF_STRING -# Install CNTK custom MKL, version 3 -RUN mkdir /usr/local/CNTKCustomMKL && \ - wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-3.tgz | \ +# Install CNTK custom MKL +RUN CNTK_CUSTOM_MKL_VERSION=3 && \ + mkdir /usr/local/CNTKCustomMKL && \ + wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-$CNTK_CUSTOM_MKL_VERSION.tgz | \ tar -xzf - -C /usr/local/CNTKCustomMKL # Install Kaldi diff --git a/Tools/docker/CNTK-GPU-1bit-Image/Dockerfile b/Tools/docker/CNTK-GPU-1bit-Image/Dockerfile index a576048dcc5b..34e2087b1eb6 100644 --- a/Tools/docker/CNTK-GPU-1bit-Image/Dockerfile +++ b/Tools/docker/CNTK-GPU-1bit-Image/Dockerfile @@ -111,9 +111,10 @@ RUN PROTOBUF_VERSION=3.1.0 \ cd .. && \ rm -rf $PROTOBUF_STRING -# Install CNTK custom MKL, version 3 -RUN mkdir /usr/local/CNTKCustomMKL && \ - wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-3.tgz | \ +# Install CNTK custom MKL +RUN CNTK_CUSTOM_MKL_VERSION=3 && \ + mkdir /usr/local/CNTKCustomMKL && \ + wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-$CNTK_CUSTOM_MKL_VERSION.tgz | \ tar -xzf - -C /usr/local/CNTKCustomMKL # Install Kaldi diff --git a/Tools/docker/CNTK-GPU-Image/Dockerfile b/Tools/docker/CNTK-GPU-Image/Dockerfile index 4f15cda1e5c9..2bca765f7748 100644 --- a/Tools/docker/CNTK-GPU-Image/Dockerfile +++ b/Tools/docker/CNTK-GPU-Image/Dockerfile @@ -92,9 +92,10 @@ RUN PROTOBUF_VERSION=3.1.0 \ cd .. && \ rm -rf $PROTOBUF_STRING -# Install CNTK custom MKL, version 3 -RUN mkdir /usr/local/CNTKCustomMKL && \ - wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-3.tgz | \ +# Install CNTK custom MKL +RUN CNTK_CUSTOM_MKL_VERSION=3 && \ + mkdir /usr/local/CNTKCustomMKL && \ + wget --no-verbose -O - https://www.cntk.ai/mkl/CNTKCustomMKL-Linux-$CNTK_CUSTOM_MKL_VERSION.tgz | \ tar -xzf - -C /usr/local/CNTKCustomMKL # Install Kaldi From f9292628b6e7a614353da27734b4c96ab10f7ffa Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Wed, 11 Jan 2017 17:45:40 +0100 Subject: [PATCH 098/120] variouse bug fixes, environment variable settings, etc. Removed dead code --- Tools/devInstall/Windows/DevInstall.ps1 | 14 +++- Tools/devInstall/Windows/helper/Action.ps1 | 81 +++++++++++-------- Tools/devInstall/Windows/helper/Common.ps1 | 14 +++- Tools/devInstall/Windows/helper/Display.ps1 | 13 ++- Tools/devInstall/Windows/helper/Download.ps1 | 28 ++----- .../devInstall/Windows/helper/Operations.ps1 | 12 ++- .../Windows/helper/PreRequisites.ps1 | 29 ++++--- .../Windows/helper/Verification.ps1 | 4 +- 8 files changed, 115 insertions(+), 80 deletions(-) diff --git a/Tools/devInstall/Windows/DevInstall.ps1 b/Tools/devInstall/Windows/DevInstall.ps1 index e3a1ecd62c77..fd3ca35ac1d7 100644 --- a/Tools/devInstall/Windows/DevInstall.ps1 +++ b/Tools/devInstall/Windows/DevInstall.ps1 @@ -1,4 +1,8 @@ -<# +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +<# .SYNOPSIS Use this cmdlet to install a CNTK development environment on your machine @@ -126,12 +130,18 @@ Function main ActionOperations $operationList - #DisplayEnd + DisplayEnd } } catch { Write-Host "Exception caught - function main / failure" Write-Host ($Error[0]).Exception + Write-Host + Write-Host " +If you want to restart this installation script, please close this Powershell Instance and start a new Powershell instance. +This will guarantee that you pick up all changes to the environment already performed by the script. + +" } } diff --git a/Tools/devInstall/Windows/helper/Action.ps1 b/Tools/devInstall/Windows/helper/Action.ps1 index f1ed9a39a5f9..0a5d6970974f 100644 --- a/Tools/devInstall/Windows/helper/Action.ps1 +++ b/Tools/devInstall/Windows/helper/Action.ps1 @@ -87,14 +87,15 @@ function InstallYml( $env= $table["Env"] $ymlFile = $table["ymlFile"] - $envsDir = join-path $basePath "envs" - $targetDir = join-path $envsDir $env + $envsDir = Join-Path $basePath "envs" + $targetDir = Join-Path $envsDir $env + $condaPath = Join-Path $basePath "Scripts\conda.exe" if (test-path -path $targetDir -PathType Container) { - $newTable = @{ Function = "InstallExe"; Command = "$basepath\Scripts\conda.exe"; Param = "env update --file $ymlFile --name $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } + $newTable = @{ Function = "InstallExe"; Command = $condaPath; Param = "env update --file `"$ymlFile`" --name $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } } else { - $newTable = @{ Function = "InstallExe"; Command = "$basepath\Scripts\conda.exe"; Param = "env create --file $ymlFile --prefix $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } + $newTable = @{ Function = "InstallExe"; Command = $condaPath; Param = "env create --file `"$ymlFile`" --prefix $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } } InstallExe $newTable @@ -165,8 +166,10 @@ function InstallMSI( $msi = $table["MsiName"] $dir = $table["MsiDir"] + $completeMsiName = Join-Path $dir $msi + $cmd = "c:\Windows\System32\MSIEXEC.EXE" - $param= "/i $dir\$msi /quiet /norestart" + $param= "/i `"$completeMsiName`" /quiet /norestart" DoProcess -doExecute $Execute -command $cmd -param "$param" -requiresRunAs $true -maxErrorLevel 0 -throwOnError $true } @@ -186,17 +189,6 @@ function MakeDirectory( } } -function RobocopyFromLocalCache( - [Parameter(Mandatory = $true)][hashtable] $table -) -{ - FunctionIntro $table - $source = $table["Source"] - $destination = $table["Destination"] - - RobocopySourceDestination $source $destination -} - function RobocopySourceDestination( [Parameter(Mandatory = $true)][string] $source, [Parameter(Mandatory = $true)][string] $destination, @@ -206,13 +198,12 @@ function RobocopySourceDestination( throw SourceDirectory [$source] is missing } - $option = "/NFL /copy:DT /dcopy:D /xj" if (-not $copyAdditive) { $option += " /MIR " } - $param = "$source $destination $option" + $param = "`"$source`" `"$destination`" $option" DoProcess -doExecute $Execute -command $roboCopyCmd -param $param -maxErrorLevel 8 -throwOnError $true } @@ -238,7 +229,7 @@ function SetEnvironmentVariable( Write-Verbose "$demoMessage[$func]: [$name] = [$content] in [$location]" } - SetEnvVar -name "$name" -content "$content" + SetUserEnvironmentVariable -name "$name" -content "$content" } function AddToPath( @@ -275,7 +266,7 @@ function AddToPath( $pathvalue = "$pathvalue;$dir" } if ($Execute) { - SetEnvVar -name $env -content "$pathvalue" + SetUserEnvironmentVariable -name $env -content $pathvalue } } @@ -396,25 +387,38 @@ function ExtractAllFromTarGz( return } - $appDir = join-path $env:ProgramFiles "git\usr\bin" - $tarApp = "tar.exe" + $app = CallGetCommand -application "git.exe" + + if ($app.length -eq 0) { + throw "Unpacking the file [$targzFileName] requires extraction utility TAR.EXE.\n Make sure GIT is installed on your maschine." + } + + $location = Get-Command "git.exe" -CommandType Application + $location = $location.Source + $location = Split-Path $location -Parent + $location = Split-Path $location -Parent + + $appDir = Join-Path $location "usr\bin" + $completeApp = Join-Path $appDir "tar.exe" - if (-not (test-path -path "$appDir\$tarApp" -PathType Leaf)) { - throw "Unpacking the file [$targzFileName] requires extraction utility [$appDir\$tarApp].\n The utility wasn't found" + if (-not (test-path -path "$completeApp" -PathType Leaf)) { + throw "Unpacking the file [$targzFileName] requires extraction utility [$completeApp].\n The utility wasn't found" } - Copy-Item $sourceFile "$destination\$targzFileName" -ErrorAction SilentlyContinue + $compleDestination = Join-Path $destination $targzFileName + + Copy-Item $sourceFile $compleDestination -ErrorAction SilentlyContinue $dosCommand = @" -set path="$appDir";%PATH% & tar.exe -xz --force-local -f "$destination\$targzFileName" -C "$destination" +set path="$appDir";%PATH% & tar.exe -xz --force-local -f "$compleDestination" -C "$destination" "@ & cmd /c $dosCommand if ($LASTEXITCODE -gt 0) { - throw "Running [$appDir\tar.exe] Command failed with exit code $LASTEXITCODE" + throw "Running [$completeApp] Command failed with exit code $LASTEXITCODE" } - Remove-Item "$destination\$targzFileName" -ErrorAction SilentlyContinue + Remove-Item "$compleDestination" -ErrorAction SilentlyContinue } function CreateBatch( @@ -593,15 +597,26 @@ function DoProcess( } } -function SetEnvVar( + +function SetUserEnvironmentVariable( + [Parameter(Mandatory=$true)][string] $name, + [Parameter(Mandatory=$true)][string] $content) +{ + Write-Verbose "SetUserEnvironmentVariable [$name] with [$content]" + + if ($Execute) { + [environment]::SetEnvironmentVariable($name, $content, "USER") + } +} + +function SetMachineEnvironmentVariable( [Parameter(Mandatory=$true)][string] $name, - [Parameter(Mandatory=$true)][string] $content, - [string] $location = "Machine") + [Parameter(Mandatory=$true)][string] $content) { - Write-Verbose "SetEnvVar [$name] with [$content]" + Write-Verbose "SetMachineEnvironmentVariable [$name] with [$content]" if ($Execute) { - $commandString = "& { [environment]::SetEnvironmentVariable('"+$name+"', '"+$content+"', '"+$location+"') }" + $commandString = "& { [environment]::SetEnvironmentVariable(`"$name`", `"$content`", `"MACHINE`") }" RunPowershellCommand -command "$commandString" -elevated $true -maxErrorLevel 0 } } diff --git a/Tools/devInstall/Windows/helper/Common.ps1 b/Tools/devInstall/Windows/helper/Common.ps1 index 348b16c95ab8..8e96f3b6d326 100644 --- a/Tools/devInstall/Windows/helper/Common.ps1 +++ b/Tools/devInstall/Windows/helper/Common.ps1 @@ -2,7 +2,6 @@ # Copyright (c) Microsoft. All rights reserved. # Licensed under the MIT license. See LICENSE.md file in the project root for full license information. # - function GetTableDefaultBool( [hashtable] $table, [string] $entryName, @@ -39,4 +38,15 @@ function GetTableDefaultInt( return $table[$entryName] } -# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file +function GetEnvironmentVariableContent( + [string] $envVarName) +{ + $envContent = [environment]::GetEnvironmentVariable($envVarName, "USER") + if ($envContent) { + return $envContent + } + $envContent = [environment]::GetEnvironmentVariable($envVarName, "MACHINE") + return $envContent +} + +# vim:set expandtab shiftwidth=2 tabstop=2:q diff --git a/Tools/devInstall/Windows/helper/Display.ps1 b/Tools/devInstall/Windows/helper/Display.ps1 index 27e84355a908..0d374280c35d 100644 --- a/Tools/devInstall/Windows/helper/Display.ps1 +++ b/Tools/devInstall/Windows/helper/Display.ps1 @@ -1,4 +1,8 @@ -function FunctionIntro( +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function FunctionIntro( [Parameter(Mandatory = $true)][hashtable] $table ) { @@ -110,7 +114,12 @@ function DisplayStart() Function DisplayEnd() { - # empty by design + Write-Host " + +Installation finished. +Please close all Powershell windows to pick up any changes to environment variables in Powershell. + +" } function DisplayAfterVerify( diff --git a/Tools/devInstall/Windows/helper/Download.ps1 b/Tools/devInstall/Windows/helper/Download.ps1 index 911f8d5c404b..2057d39b7729 100644 --- a/Tools/devInstall/Windows/helper/Download.ps1 +++ b/Tools/devInstall/Windows/helper/Download.ps1 @@ -1,4 +1,8 @@ -function DownloadOperations( +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function DownloadOperations( [parameter(Mandatory=$true)][array] $downloadList) { Write-Host "Performing download operations" @@ -89,28 +93,6 @@ function LocalCopyFile( copy-Item $source $destination -Force -ErrorAction SilentlyContinue } -function RobocopyFromServer( - [Parameter(Mandatory = $true)][hashtable] $table -) -{ - FunctionIntro $table - - $source = $table["Source"] - $destination = $table["Destination"] - - $source = $LocalServer+"\$source" - - if (-not (test-path $source)) { - throw "SourceDirectory [$source] is missing" - } - - $option = "/MIR /NFL /copy:DT /dcopy:D /xj" - - $param = "$source $destination $option" - - DoProcess -command $roboCopyCmd -param "$param" -IgnoreNonZeroExitCode $true -} - function NotImplemented( [Parameter(Mandatory = $true)][hashtable] $table ) diff --git a/Tools/devInstall/Windows/helper/Operations.ps1 b/Tools/devInstall/Windows/helper/Operations.ps1 index 6484cd50c170..ecbdca3239cd 100644 --- a/Tools/devInstall/Windows/helper/Operations.ps1 +++ b/Tools/devInstall/Windows/helper/Operations.ps1 @@ -1,4 +1,8 @@ -function OpAnaconda3411( +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function OpAnaconda3411( [parameter(Mandatory=$true)][string] $cache, [parameter(Mandatory=$true)][string] $AnacondaBasePath) { @@ -13,7 +17,7 @@ @( @{ShortName = "ANA3-411"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath; } ); Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); - Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/InstallationType=JustMe /AddToPath=0 /RegisterPython=0 /S /D=$targetPath"; runAs=$false; Message = ".... This will take some time. Please be patient ...." } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/InstallationType=JustMe /AddToPath=0 /RegisterPython=0 /S /D=`"$targetPath`""; runAs=$false; Message = ".... This will take some time. Please be patient ...." } ); } ) } @@ -62,7 +66,7 @@ function OpBoost160VS15( Download = @( @{Function = "Download"; Method = "WebClient"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $targetPath }, @{Function = "SetEnvironmentVariable"; EnvVar = $envVarLib; Content = $envContentLib }, - @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=$targetPath /SP- /SILENT /NORESTART"; runAs=$false } ); + @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=`"$targetPath`" /SP- /SILENT /NORESTART"; runAs=$false } ); } ) } @@ -133,7 +137,7 @@ function OpMSMPI70SDK( Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI SDK \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); #Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft MPI SDK \(\d+\."; Compare = "^Microsoft MPI SDK \(7\.0\.12437\.6\)"; MatchExact = $false } ); Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$localFilename"; ExpectedSize = $downloadSize } ); - Action = @( @{Function = "InstallMsi"; MsiName = "$localFilename" ; MsiDir = "$cache" } ) + Action = @( @{Function = "InstallMsi"; MsiName = "$localFilename" ; MsiDir = "$cache" } ) } ) } diff --git a/Tools/devInstall/Windows/helper/PreRequisites.ps1 b/Tools/devInstall/Windows/helper/PreRequisites.ps1 index e88973184251..62ab966bc36e 100644 --- a/Tools/devInstall/Windows/helper/PreRequisites.ps1 +++ b/Tools/devInstall/Windows/helper/PreRequisites.ps1 @@ -5,13 +5,16 @@ function PreReqOperations( [array] $actionList = @()) { - foreach ($item in $actionList) { + $continueInstallation = $true + foreach ($item in $actionList) { foreach ($prereqItem in $item.PreReq) { - PreRequisiteItem $prereqItem + $continueInstallation = $continueInstallation -and (PreRequisiteItem $prereqItem) } } - - Write-Host "Install operations finished" + if (-not $continueInstallation) { + throw "Not all pre-requisites installed, installation terminated." + } + Write-Host "Checking pre-requisites finished" Write-Host } @@ -36,6 +39,7 @@ function PrereqInfo2013Up5( We require the installation of Visual Studio 2013 Update 5 to continue. " + return $false } function PrereqInfoVS15( @@ -43,13 +47,14 @@ function PrereqInfoVS15( ) { FunctionIntro $table - Write-Host " + Write-Warning " -Installation of Visual Studio 2015 Update 3 is a pre-requisite before installation -can continue. Please check [https://www.visualstudio.com/vs/] for more details on -Visual Studion 2015 +Installation of Visual Studio 2015 Update 3 is a pre-requisite before installation can continue. +Please check + https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows +for more details. " -return + return $false } function PrereqInfoCuda8( @@ -57,11 +62,13 @@ function PrereqInfoCuda8( ) { FunctionIntro $table - Write-Host " + Write-Warning " Installation of NVidia CUDA 8.0 is a pre-requisite before installation can continue. +Please check + https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows +for more details. " -return } function StopInstallation( diff --git a/Tools/devInstall/Windows/helper/Verification.ps1 b/Tools/devInstall/Windows/helper/Verification.ps1 index f91fd8bfd46f..fdb402635e97 100644 --- a/Tools/devInstall/Windows/helper/Verification.ps1 +++ b/Tools/devInstall/Windows/helper/Verification.ps1 @@ -280,9 +280,7 @@ function VerifyEnvironmentAndData( $content = $table["Content"] $location = "User" - $path = Join-Path "env:" $name - $envContent = Get-ChildItem $path -ErrorAction SilentlyContinue - $envContent = $envContent.Value + $envContent = GetEnvironmentVariableContent $name $noInstallRequired = ($envContent -ieq $content) Write-Verbose "[$func]: [$name] == [$content] returned [$noInstallRequired]" From 89f9a54eca6249c6bcf5187861612401929d2871 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Wed, 11 Jan 2017 18:54:00 +0100 Subject: [PATCH 099/120] addressed CR comments --- Tools/devInstall/Windows/DevInstall.ps1 | 5 ----- Tools/devInstall/Windows/helper/Action.ps1 | 9 +++------ Tools/devInstall/Windows/helper/Common.ps1 | 7 +------ Tools/devInstall/Windows/helper/Display.ps1 | 1 - .../devInstall/Windows/helper/PreRequisites.ps1 | 17 ++--------------- .../devInstall/Windows/helper/Verification.ps1 | 2 +- 6 files changed, 7 insertions(+), 34 deletions(-) diff --git a/Tools/devInstall/Windows/DevInstall.ps1 b/Tools/devInstall/Windows/DevInstall.ps1 index fd3ca35ac1d7..c92055c0b301 100644 --- a/Tools/devInstall/Windows/DevInstall.ps1 +++ b/Tools/devInstall/Windows/DevInstall.ps1 @@ -137,11 +137,6 @@ Function main Write-Host "Exception caught - function main / failure" Write-Host ($Error[0]).Exception Write-Host - Write-Host " -If you want to restart this installation script, please close this Powershell Instance and start a new Powershell instance. -This will guarantee that you pick up all changes to the environment already performed by the script. - -" } } diff --git a/Tools/devInstall/Windows/helper/Action.ps1 b/Tools/devInstall/Windows/helper/Action.ps1 index 0a5d6970974f..ac85e6d5f967 100644 --- a/Tools/devInstall/Windows/helper/Action.ps1 +++ b/Tools/devInstall/Windows/helper/Action.ps1 @@ -15,9 +15,6 @@ function ActionOperations( ActionItem $actionItem } } - - Write-Host "Install operations finished" - Write-Host } function ActionItem( @@ -87,9 +84,9 @@ function InstallYml( $env= $table["Env"] $ymlFile = $table["ymlFile"] - $envsDir = Join-Path $basePath "envs" + $envsDir = Join-Path $basePath envs $targetDir = Join-Path $envsDir $env - $condaPath = Join-Path $basePath "Scripts\conda.exe" + $condaPath = Join-Path $basePath Scripts\conda.exe if (test-path -path $targetDir -PathType Container) { $newTable = @{ Function = "InstallExe"; Command = $condaPath; Param = "env update --file `"$ymlFile`" --name $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } @@ -389,7 +386,7 @@ function ExtractAllFromTarGz( $app = CallGetCommand -application "git.exe" - if ($app.length -eq 0) { + if ($app) { throw "Unpacking the file [$targzFileName] requires extraction utility TAR.EXE.\n Make sure GIT is installed on your maschine." } diff --git a/Tools/devInstall/Windows/helper/Common.ps1 b/Tools/devInstall/Windows/helper/Common.ps1 index 8e96f3b6d326..f09d3e0face1 100644 --- a/Tools/devInstall/Windows/helper/Common.ps1 +++ b/Tools/devInstall/Windows/helper/Common.ps1 @@ -41,12 +41,7 @@ function GetTableDefaultInt( function GetEnvironmentVariableContent( [string] $envVarName) { - $envContent = [environment]::GetEnvironmentVariable($envVarName, "USER") - if ($envContent) { - return $envContent - } - $envContent = [environment]::GetEnvironmentVariable($envVarName, "MACHINE") - return $envContent + return [environment]::GetEnvironmentVariable($envVarName) } # vim:set expandtab shiftwidth=2 tabstop=2:q diff --git a/Tools/devInstall/Windows/helper/Display.ps1 b/Tools/devInstall/Windows/helper/Display.ps1 index 0d374280c35d..072285be5ec1 100644 --- a/Tools/devInstall/Windows/helper/Display.ps1 +++ b/Tools/devInstall/Windows/helper/Display.ps1 @@ -117,7 +117,6 @@ Function DisplayEnd() Write-Host " Installation finished. -Please close all Powershell windows to pick up any changes to environment variables in Powershell. " } diff --git a/Tools/devInstall/Windows/helper/PreRequisites.ps1 b/Tools/devInstall/Windows/helper/PreRequisites.ps1 index 62ab966bc36e..a437f6cdd44b 100644 --- a/Tools/devInstall/Windows/helper/PreRequisites.ps1 +++ b/Tools/devInstall/Windows/helper/PreRequisites.ps1 @@ -8,7 +8,8 @@ function PreReqOperations( $continueInstallation = $true foreach ($item in $actionList) { foreach ($prereqItem in $item.PreReq) { - $continueInstallation = $continueInstallation -and (PreRequisiteItem $prereqItem) + $continueInstallation = $false + PreRequisiteItem $prereqItem } } if (-not $continueInstallation) { @@ -29,19 +30,6 @@ function PreRequisiteItem( Invoke-Expression $expr } -function PrereqInfo2013Up5( - [Parameter(Mandatory = $true)][hashtable] $table -) -{ - FunctionIntro $table - - Write-Host " - -We require the installation of Visual Studio 2013 Update 5 to continue. -" - return $false -} - function PrereqInfoVS15( [Parameter(Mandatory = $true)][hashtable] $table ) @@ -54,7 +42,6 @@ Please check https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows for more details. " - return $false } function PrereqInfoCuda8( diff --git a/Tools/devInstall/Windows/helper/Verification.ps1 b/Tools/devInstall/Windows/helper/Verification.ps1 index fdb402635e97..dcaf0422ae1d 100644 --- a/Tools/devInstall/Windows/helper/Verification.ps1 +++ b/Tools/devInstall/Windows/helper/Verification.ps1 @@ -281,7 +281,7 @@ function VerifyEnvironmentAndData( $location = "User" $envContent = GetEnvironmentVariableContent $name - $noInstallRequired = ($envContent -ieq $content) + $noInstallRequired = $envContent -eq $content Write-Verbose "[$func]: [$name] == [$content] returned [$noInstallRequired]" return $noInstallRequired From c1684a7a4ece205df3375ddca27fdb70585b81a0 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Wed, 11 Jan 2017 19:17:41 +0100 Subject: [PATCH 100/120] more cr changes --- Tools/devInstall/Windows/helper/Action.ps1 | 16 ++++++++-------- Tools/devInstall/Windows/helper/Common.ps1 | 9 +++++++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/Tools/devInstall/Windows/helper/Action.ps1 b/Tools/devInstall/Windows/helper/Action.ps1 index ac85e6d5f967..37dc2fd9b7ee 100644 --- a/Tools/devInstall/Windows/helper/Action.ps1 +++ b/Tools/devInstall/Windows/helper/Action.ps1 @@ -384,10 +384,10 @@ function ExtractAllFromTarGz( return } - $app = CallGetCommand -application "git.exe" + $app = CallGetCommand -application git.exe if ($app) { - throw "Unpacking the file [$targzFileName] requires extraction utility TAR.EXE.\n Make sure GIT is installed on your maschine." + throw "Unpacking the file [$targzFileName] requires extraction utility TAR.EXE.\n Make sure `"Git for Windows`" is installed on your machine." } $location = Get-Command "git.exe" -CommandType Application @@ -398,24 +398,24 @@ function ExtractAllFromTarGz( $appDir = Join-Path $location "usr\bin" $completeApp = Join-Path $appDir "tar.exe" - if (-not (test-path -path "$completeApp" -PathType Leaf)) { + if (-not (Test-Path -path $completeApp -PathType Leaf)) { throw "Unpacking the file [$targzFileName] requires extraction utility [$completeApp].\n The utility wasn't found" } - $compleDestination = Join-Path $destination $targzFileName + $completeDestination = Join-Path $destination $targzFileName - Copy-Item $sourceFile $compleDestination -ErrorAction SilentlyContinue + Copy-Item $sourceFile $completeDestination -ErrorAction SilentlyContinue $dosCommand = @" -set path="$appDir";%PATH% & tar.exe -xz --force-local -f "$compleDestination" -C "$destination" +set path=$appDir;%PATH% & tar.exe -xz --force-local -f "$completeDestination" -C "$destination" "@ & cmd /c $dosCommand if ($LASTEXITCODE -gt 0) { throw "Running [$completeApp] Command failed with exit code $LASTEXITCODE" } - - Remove-Item "$compleDestination" -ErrorAction SilentlyContinue + + Remove-Item "$completeDestination" -ErrorAction SilentlyContinue } function CreateBatch( diff --git a/Tools/devInstall/Windows/helper/Common.ps1 b/Tools/devInstall/Windows/helper/Common.ps1 index f09d3e0face1..89ab9e45bce1 100644 --- a/Tools/devInstall/Windows/helper/Common.ps1 +++ b/Tools/devInstall/Windows/helper/Common.ps1 @@ -41,7 +41,12 @@ function GetTableDefaultInt( function GetEnvironmentVariableContent( [string] $envVarName) { - return [environment]::GetEnvironmentVariable($envVarName) + $envContent = [environment]::GetEnvironmentVariable($envVarName, "USER") + if ($envContent) { + return $envContent + } + $envContent = [environment]::GetEnvironmentVariable($envVarName, "MACHINE") + return $envContent } -# vim:set expandtab shiftwidth=2 tabstop=2:q +# vim:set expandtab shiftwidth=2 tabstop=2: From f1600855501906a52d705080563cffb13b975ac9 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Wed, 11 Jan 2017 19:30:50 +0100 Subject: [PATCH 101/120] addressed CR comments --- Tools/devInstall/Windows/helper/Action.ps1 | 2 +- Tools/devInstall/Windows/helper/Display.ps1 | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Tools/devInstall/Windows/helper/Action.ps1 b/Tools/devInstall/Windows/helper/Action.ps1 index 37dc2fd9b7ee..c981e8bd45ab 100644 --- a/Tools/devInstall/Windows/helper/Action.ps1 +++ b/Tools/devInstall/Windows/helper/Action.ps1 @@ -386,7 +386,7 @@ function ExtractAllFromTarGz( $app = CallGetCommand -application git.exe - if ($app) { + if (-not $app) { throw "Unpacking the file [$targzFileName] requires extraction utility TAR.EXE.\n Make sure `"Git for Windows`" is installed on your machine." } diff --git a/Tools/devInstall/Windows/helper/Display.ps1 b/Tools/devInstall/Windows/helper/Display.ps1 index 072285be5ec1..30b1144efa18 100644 --- a/Tools/devInstall/Windows/helper/Display.ps1 +++ b/Tools/devInstall/Windows/helper/Display.ps1 @@ -117,7 +117,6 @@ Function DisplayEnd() Write-Host " Installation finished. - " } From 546e1d29dc2803fbc712df58dae9ae5f95c3717e Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Mon, 9 Jan 2017 00:51:53 -0800 Subject: [PATCH 102/120] CNTK v2 library: UserDefined function simplifications a) Add support for saving forward compute values in base BackPropState type itself b) Function::Forward override now receives values for all inputs including Parameters and Constants --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 64 +++++++++++++++---- Source/CNTKv2LibraryDll/CompositeFunction.h | 16 +++-- Source/CNTKv2LibraryDll/Function.cpp | 39 +++++++++++ Source/CNTKv2LibraryDll/PrimitiveFunction.h | 12 +--- .../V2LibraryTests/FunctionTests.cpp | 6 +- .../UnitTests/V2LibraryTests/TensorTests.cpp | 4 +- .../UserDefinedFunctionTests.cpp | 20 ++++-- bindings/csharp/Swig/cntk_cs.i | 4 ++ 8 files changed, 129 insertions(+), 36 deletions(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index e92aa8f4b749..51bcc60e9e46 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -2303,21 +2303,41 @@ namespace CNTK class BackPropState : public std::enable_shared_from_this { public: + /// + /// Constructs a BackPropState object + /// The function and computeDevice parameters record the Function and compute device that 'this' BackPropState corresponds to + /// The forwardPropValuesToSave is an optional map of forward compute values saved for later use during back propagation of gradients + /// in the backward call that 'this' BackPropState object is used. + /// + BackPropState(const FunctionPtr& function, const DeviceDescriptor& computeDevice, const std::unordered_map& forwardPropValuesToSave = {}) + : m_function(function), m_forwardComputeDevice(computeDevice), m_savedForwardPropValues(forwardPropValuesToSave) + {} + + /// + /// Destructor + /// + virtual ~BackPropState() {} + /// /// Returns the Function that 'this' BackPropState belongs to /// FunctionPtr Function() const { return m_function; } + + /// + /// Returns the DeviceDescriptor that the forward call, that created 'this' BackPropState, was executed on + /// DeviceDescriptor Device() const { return m_forwardComputeDevice; } - virtual ~BackPropState() {} - protected: - BackPropState(const FunctionPtr& function, const DeviceDescriptor& computeDevice) - : m_function(function), m_forwardComputeDevice(computeDevice) - {} + /// + /// Returns the forward prop values saved when constructing 'this' BackPropState state + /// for later use during back propagation of gradients in a backward call that 'this' BackPropState object is used. + /// + const std::unordered_map& SavedForwardPropValues() const { return m_savedForwardPropValues; } protected: FunctionPtr m_function; DeviceDescriptor m_forwardComputeDevice; + std::unordered_map m_savedForwardPropValues; }; typedef std::shared_ptr BackPropStatePtr; @@ -2373,10 +2393,10 @@ namespace CNTK /// and the user is responsible for ensuring that the contents of the inputs and outputs are unchanged until after any uses of the BackPropState instance /// for backpropagating gradients through this Function. /// - virtual BackPropStatePtr Forward(const std::unordered_map& arguments, - std::unordered_map& outputs, - const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice(), - const std::unordered_set& outputsToRetainBackwardStateFor = {}) = 0; + CNTK_API BackPropStatePtr Forward(const std::unordered_map& arguments, + std::unordered_map& outputs, + const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice(), + const std::unordered_set& outputsToRetainBackwardStateFor = {}); /// /// Backpropagates supplied 'rootGradientValues' for one or more of the output variables of the Function, to produce gradient Values @@ -2387,15 +2407,35 @@ namespace CNTK /// The 'state' parameter is an instance of an BackPropState instance obtained from a previous call to the Forward method on 'this; Function for the /// computation that this gradient backpropagation corresponds to. /// - virtual void Backward(const BackPropStatePtr& state, - const std::unordered_map& rootGradientValues, - std::unordered_map& backPropagatedGradientValuesForInputs) = 0; + CNTK_API virtual void Backward(const BackPropStatePtr& state, + const std::unordered_map& rootGradientValues, + std::unordered_map& backPropagatedGradientValuesForInputs); /// /// Returns the name of the operation that this Function denotes /// virtual const std::wstring& OpName() const = 0; + protected: + /// + /// Computes and stores the values of specified variables in the 'outputs' map, using provided 'inputs' values for each input of the Function. + /// The variables specified in the 'outputs' map denote the subset of 'this' Function's output variables that the caller wants to obtain values of. + /// Callers may specify the storage to be used for storing the 'outputs' Values or pass null in which case the implementation allocates the actual storage + /// for the 'outputs' for which the ValuePtr mapping was left null by the caller. + /// The optional 'outputsToRetainBackwardStateFor' parameter specifies the subset of the Function's output variables for which gradients will be specified + /// in a subsequent Backward call for backpropagation. + /// The method returns a BackPropState object containing all intermediate variable values needed during backpropagation of gradients from the + /// 'outputsToRetainBackwardStateFor' outputs of the Function to any of the inputs of the Function, in a subsequent Backward call. + /// Note that the returned BackPropState instance also stores a reference to the supplied 'inputs' Values and generated 'outputs' Values + /// and the user is responsible for ensuring that the contents of the inputs and outputs are unchanged until after any uses of the BackPropState instance + /// for backpropagating gradients through this Function. + /// User defined Functions that derive from the Function type must implement this method. + /// + virtual BackPropStatePtr Forward(const std::vector& inputValues, + std::unordered_map& outputs, + const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice(), + const std::unordered_set& outputsToRetainBackwardStateFor = {}) = 0; + public: // Optional overrides diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.h b/Source/CNTKv2LibraryDll/CompositeFunction.h index c6ccba422bf9..9a4e1ba9151e 100644 --- a/Source/CNTKv2LibraryDll/CompositeFunction.h +++ b/Source/CNTKv2LibraryDll/CompositeFunction.h @@ -74,10 +74,18 @@ namespace CNTK return MakeSharedObject(rootFunction, std::move(visitedFunctions), name, uid); } - virtual BackPropStatePtr Forward(const std::unordered_map& arguments, - std::unordered_map& outputs, - const DeviceDescriptor& computeDevice, - const std::unordered_set& outputsToRetainBackwardStateFor) override; + BackPropStatePtr Forward(const std::unordered_map& arguments, + std::unordered_map& outputs, + const DeviceDescriptor& computeDevice, + const std::unordered_set& outputsToRetainBackwardStateFor); + + virtual BackPropStatePtr Forward(const std::vector& /*inputValues*/, + std::unordered_map& /*outputs*/, + const DeviceDescriptor& /*computeDevice*/, + const std::unordered_set& /*outputsToRetainBackwardStateFor*/) + { + NOT_IMPLEMENTED; + } virtual void Backward(const BackPropStatePtr& state, const std::unordered_map& rootGradientValues, diff --git a/Source/CNTKv2LibraryDll/Function.cpp b/Source/CNTKv2LibraryDll/Function.cpp index 8573c1358c0c..98421a755e65 100644 --- a/Source/CNTKv2LibraryDll/Function.cpp +++ b/Source/CNTKv2LibraryDll/Function.cpp @@ -68,6 +68,45 @@ namespace CNTK /*virtual*/ Function::~Function() {} + BackPropStatePtr Function::Forward(const std::unordered_map& arguments, + std::unordered_map& outputs, + const DeviceDescriptor& computeDevice, + const std::unordered_set& outputsToRetainBackwardStateFor) + { + auto compositeFunction = dynamic_cast(this); + if (compositeFunction) + return compositeFunction->Forward(arguments, outputs, computeDevice, outputsToRetainBackwardStateFor); + + std::vector inputValues; + auto functionInputs = Inputs(); + for (const auto& input : functionInputs) + { + ValuePtr inputValue; + if (input.IsConstant()) + inputValue = MakeSharedObject(Constant(input).Value()); + else if (input.IsParameter()) + inputValue = MakeSharedObject(Parameter(input).Value()); + else + { + if (arguments.find(input) == arguments.end()) + InvalidArgument("Function::Forward: No value specified for input variable (Name=%S, Uid=%S)", input.Name().c_str(), input.Uid().c_str()); + + inputValue = arguments.at(input); + } + + inputValues.push_back(inputValue); + } + + return Forward(inputValues, outputs, computeDevice, outputsToRetainBackwardStateFor); + } + + /*virtual*/ void Function::Backward(const BackPropStatePtr& /*state*/, + const std::unordered_map& /*rootGradientValues*/, + std::unordered_map& /*backPropagatedGradientValuesForInputs*/) + { + NOT_IMPLEMENTED; + } + void Function::SetName(const std::wstring& name) { if (!Name().empty() && !Internal::IsRenamingFunctionsAllowed()) diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.h b/Source/CNTKv2LibraryDll/PrimitiveFunction.h index 4067052738f0..05ba9322684d 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.h +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.h @@ -231,17 +231,11 @@ namespace CNTK : PrimitiveFunction(op, inputs, std::move(functionConfig), functionName, GenerateUid(op)) {} - virtual BackPropStatePtr Forward(const std::unordered_map& /*arguments*/, + // Primitive functions are currently implemented using the core CNTK engine ComputationNode types + virtual BackPropStatePtr Forward(const std::vector& /*inputValues*/, std::unordered_map& /*outputs*/, const DeviceDescriptor& /*computeDevice*/, - const std::unordered_set& /*outputsToRetainBackwardStateFor*/) override - { - NOT_IMPLEMENTED; - } - - virtual void Backward(const BackPropStatePtr& /*state*/, - const std::unordered_map& /*rootGradientValues*/, - std::unordered_map& /*backPropagatedGradientValuesForInputs*/) override + const std::unordered_set& /*outputsToRetainBackwardStateFor*/) { NOT_IMPLEMENTED; } diff --git a/Tests/UnitTests/V2LibraryTests/FunctionTests.cpp b/Tests/UnitTests/V2LibraryTests/FunctionTests.cpp index ab9a6753372d..7ab6c5fbfd64 100644 --- a/Tests/UnitTests/V2LibraryTests/FunctionTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/FunctionTests.cpp @@ -531,7 +531,7 @@ void TestChangingParameterValues(size_t rank, const DeviceDescriptor& device) ValuePtr outputValue = MakeSharedObject(MakeSharedObject(shape, outputData, false)); std::unordered_map outputs = { { plus->Output(), outputValue } }; - plus->Forward({}, outputs, device); + plus->Forward(std::unordered_map({}), outputs, device); NDArrayViewPtr cpuView; auto getParameterData = [&cpuView](const Parameter& p) -> const ElementType* @@ -563,7 +563,7 @@ void TestChangingParameterValues(size_t rank, const DeviceDescriptor& device) } param.RecordValueUpdate(); - plus->Forward({}, outputs, device); + plus->Forward(std::unordered_map({}), outputs, device); parameterData = getParameterData(param); for (int i = 0; i < numElements; i++) { @@ -582,7 +582,7 @@ void TestChangingParameterValues(size_t rank, const DeviceDescriptor& device) auto newValuesNDarray = MakeSharedObject(shape, newValues, false); param.SetValue(newValuesNDarray); - plus->Forward({}, outputs, device); + plus->Forward(std::unordered_map({}), outputs, device); parameterData = getParameterData(param); for (int i = 0; i < numElements; i++) { diff --git a/Tests/UnitTests/V2LibraryTests/TensorTests.cpp b/Tests/UnitTests/V2LibraryTests/TensorTests.cpp index cff68e0b6a42..569149e1a8de 100644 --- a/Tests/UnitTests/V2LibraryTests/TensorTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/TensorTests.cpp @@ -65,7 +65,7 @@ void TestTensorPlus(size_t numAxesLeftOperand, size_t numAxesRightOperand, const std::unordered_map outputs = { { plusFunc->Output(), outputValue } }; BackPropStatePtr backPropState; if (useConstantInputsOnly) - backPropState = plusFunc->Forward({}, outputs, device, { plusFunc->Output() }); + backPropState = plusFunc->Forward(std::unordered_map({}), outputs, device, { plusFunc->Output() }); else backPropState = plusFunc->Forward({ { leftInputVar, MakeSharedObject(leftInputValue) }, { rightInputVar, MakeSharedObject(rightInputValue) } }, outputs, device, { plusFunc->Output() }); @@ -111,7 +111,7 @@ void TestInfAndNans() auto outputValue = MakeSharedObject(MakeSharedObject(NDShape(0), outputData)); std::unordered_map outputs = { { divideFunc->Output(), outputValue } }; - divideFunc->Forward({}, outputs, device); + divideFunc->Forward(std::unordered_map({}), outputs, device); if (outputData[0] != std::numeric_limits::infinity()) throw std::runtime_error("1/0 != Infinity"); diff --git a/Tests/UnitTests/V2LibraryTests/UserDefinedFunctionTests.cpp b/Tests/UnitTests/V2LibraryTests/UserDefinedFunctionTests.cpp index b607053b47d7..cd2e09035388 100644 --- a/Tests/UnitTests/V2LibraryTests/UserDefinedFunctionTests.cpp +++ b/Tests/UnitTests/V2LibraryTests/UserDefinedFunctionTests.cpp @@ -23,7 +23,7 @@ class UserDefinedTimesOrPlusFunction final : public Function public: - BackPropStatePtr Forward(const std::unordered_map& arguments, + BackPropStatePtr Forward(const std::vector& inputValues, std::unordered_map& outputs, const DeviceDescriptor& computeDevice, const std::unordered_set& outputsToRetainBackwardStateFor) override @@ -33,14 +33,22 @@ class UserDefinedTimesOrPlusFunction final : public Function if (!outputsToRetainBackwardStateFor.empty()) retainBackwardStateFor = { m_timesOrPlusFunc->Output() }; + auto inputs = Inputs(); + auto GetInputIndex = [&inputs](const Variable& input) -> size_t { + for (size_t i = 0; i < inputs.size(); ++i) + { + if (inputs[i] == input) + return i; + } + + std::runtime_error("GetInputIndex: Specified variable is not an input of this Function"); + return 0; + }; + std::unordered_map argumentValues; for (auto argumentMapping : m_timesOrPlusFuncArgumentMap) { - ValuePtr argValue; - if (argumentMapping.second.IsConstant() || argumentMapping.second.IsParameter()) - argValue = MakeSharedObject(argumentMapping.second.IsConstant() ? Constant(argumentMapping.second).Value() : Parameter(argumentMapping.second).Value()); - else - argValue = arguments.at(argumentMapping.second); + ValuePtr argValue = inputValues[GetInputIndex(argumentMapping.second)]; if (argumentMapping.first.IsParameter()) Parameter(argumentMapping.first).SetValue(argValue->Data()); diff --git a/bindings/csharp/Swig/cntk_cs.i b/bindings/csharp/Swig/cntk_cs.i index 00483dcc6cea..b4ed26cb1c8c 100644 --- a/bindings/csharp/Swig/cntk_cs.i +++ b/bindings/csharp/Swig/cntk_cs.i @@ -151,6 +151,10 @@ std::unordered_map& outputs, const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice(), const std::unordered_set& outputsToRetainBackwardStateFor = {}); +%ignore CNTK::Function::Forward(const std::vector& inputValues, + std::unordered_map& outputs, + const DeviceDescriptor& computeDevice = DeviceDescriptor::UseDefaultDevice(), + const std::unordered_set& outputsToRetainBackwardStateFor = {}); %ignore CNTK::Function::Serialize() const; %ignore CNTK::Function::Deserialize(const Dictionary& dictionary, const ::CNTK::DeviceDescriptor& device = DeviceDescriptor::UseDefaultDevice()); %ignore CNTK::Function::Parameters() const; From 0cb702334ab7e12497960d7fc5075459039b09b4 Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Mon, 9 Jan 2017 23:53:55 -0800 Subject: [PATCH 103/120] CNTK v2 library: Enable inference of output shape for zip operations even when the shapes for some of the inputs are unknown. This is useful for shape inference inside recurrence loops --- Source/CNTKv2LibraryDll/Function.cpp | 4 +- Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 146 ++++++++++-------- Source/CNTKv2LibraryDll/PrimitiveFunction.h | 14 +- .../python/cntk/ops/tests/function_tests.py | 11 +- 4 files changed, 107 insertions(+), 68 deletions(-) diff --git a/Source/CNTKv2LibraryDll/Function.cpp b/Source/CNTKv2LibraryDll/Function.cpp index 98421a755e65..e77ab45a7fc4 100644 --- a/Source/CNTKv2LibraryDll/Function.cpp +++ b/Source/CNTKv2LibraryDll/Function.cpp @@ -423,7 +423,7 @@ namespace CNTK std::unordered_map functionVisitCounts; // An arbitrary cap on changing output shape of recurrent nodes, to detect infinite inference loops - const size_t maxNumValidationPassesAllowed = 25; + const size_t maxNumValidationPassesAllowed = 128; bool recurrentNodeOutputModified = false; size_t numValidationPasses = 0; do @@ -435,7 +435,7 @@ namespace CNTK } while (recurrentNodeOutputModified && (numValidationPasses < maxNumValidationPassesAllowed)); if (numValidationPasses >= maxNumValidationPassesAllowed) - LogicError("A recurrent node output shape change happened in successive %d validation passes indicating a potential infinite inference loop!", (int)numValidationPasses); + LogicError("A recurrent node output shape change happened in max allowed (%d) successive validation passes indicating a potential infinite inference loop!", (int)numValidationPasses); for (auto replacementPair : placeholderReplacements) { diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index 0636a086ecbe..418b6439d7c1 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -73,14 +73,8 @@ namespace CNTK /*static*/ const std::wstring PrimitiveFunction::AttributeNameRngOffset = L"rngOffset"; /*static*/ const std::wstring PrimitiveFunction::AttributeNameUnpoolingWindowShape = L"unpoolingWindowShape"; - /*static*/ std::vector PrimitiveFunction::GetOutputVariables(PrimitiveOpType op, - std::vector& inputs, - Dictionary& functionConfig, - bool inferDimensions, - const std::wstring& functionName) + /*static*/ DataType PrimitiveFunction::GetOutputDataType(PrimitiveOpType op, std::vector& inputs, bool inferDimensions) { - if (op == PrimitiveOpType::Combine) - return inputs; // We use the first non-constant input operand's DataType as the output DataType // In case there are no non-constant known DataTypes, we just pick the first known operand DataType @@ -124,6 +118,11 @@ namespace CNTK } } + return outputDataType; + } + + /*static*/ std::vector PrimitiveFunction::GetOutputDynamicAxes(PrimitiveOpType op, std::vector& inputs, Dictionary& functionConfig) + { // We currently require that the inputs' dynamic axes, if any, match std::vector outputDynamicAxes; if ((op == PrimitiveOpType::SumAll) || @@ -205,11 +204,70 @@ namespace CNTK } } - NDShape outputShape; - bool areAnyInputShapesUnknown = (std::find_if(inputs.begin(), inputs.end(), [](const Variable& input) { return input.Shape().IsUnknown(); }) != inputs.end()); - if (areAnyInputShapesUnknown) - outputShape = NDShape::Unknown; - else + return outputDynamicAxes; + } + + /*static*/ std::vector PrimitiveFunction::GetOutputVariables(PrimitiveOpType op, + std::vector& inputs, + Dictionary& functionConfig, + bool inferDimensions, + const std::wstring& functionName) + { + if (op == PrimitiveOpType::Combine) + return inputs; + + DataType outputDataType = GetOutputDataType(op, inputs, inferDimensions); + std::vector outputDynamicAxes = GetOutputDynamicAxes(op, inputs, functionConfig); + + NDShape outputShape = NDShape::Unknown; + bool allInputShapesUnknown = (std::find_if(inputs.begin(), inputs.end(), [](const Variable& input) { return !input.Shape().IsUnknown(); }) == inputs.end()); + if (!allInputShapesUnknown && (outputDynamicAxes != Axis::UnknownDynamicAxes())) + { + switch (op) + { + // Elementwise operators' shapes are a zip of inputs and can be determined even if some of the input shapes are unknown + case PrimitiveOpType::Plus: + case PrimitiveOpType::LogPlus: + case PrimitiveOpType::Minus: + case PrimitiveOpType::ElementTimes: + case PrimitiveOpType::Equal: + case PrimitiveOpType::NotEqual: + case PrimitiveOpType::Less: + case PrimitiveOpType::LessEqual: + case PrimitiveOpType::Greater: + case PrimitiveOpType::GreaterEqual: + case PrimitiveOpType::PastValue: + case PrimitiveOpType::FutureValue: + { + assert(inputs.size() == 2); + if ((op == PrimitiveOpType::PastValue) || (op == PrimitiveOpType::FutureValue)) + { + Variable inputOperandVar = inputs[0]; + Variable initialStateVar = inputs[1]; + + // TODO: We currently only support input operand with 1 dynamic axis for PastValue/FutureValue + if ((inputOperandVar.DynamicAxes() != Axis::UnknownDynamicAxes()) && (inputOperandVar.DynamicAxes().size() != 2)) + LogicError("Currently PastValue/FutureValue Function only supports input operand with 2 dynamic axis (1 sequence-axis and 1 batch-axis)"); + + if (!initialStateVar.DynamicAxes().empty()) + LogicError("Currently PastValue/FutureValue Function does not support initial state operand with dynamic axes!"); + } + + outputShape = BinaryElementwiseOpOutputShape(op, inputs[0], inputs[1], true, inferDimensions); + break; + } + case PrimitiveOpType::Clip: + assert(inputs.size() == 3); + outputShape = NaryElementwiseOpOutputShape(op, inputs, true, inferDimensions); + break; + case PrimitiveOpType::Select: + assert(inputs.size() == 3); + outputShape = NaryElementwiseOpOutputShape(op, inputs, true, inferDimensions); + break; + default: + // For all other operations, shapes of all inputs must be known to determine the output shape + bool anyInputShapesUnknown = (std::find_if(inputs.begin(), inputs.end(), [](const Variable& input) { return input.Shape().IsUnknown(); }) != inputs.end()); + if (!anyInputShapesUnknown) { switch (op) { @@ -231,10 +289,21 @@ namespace CNTK case PrimitiveOpType::Sin: case PrimitiveOpType::Cos: case PrimitiveOpType::Pass: - { assert(inputs.size() == 1); outputShape = UnaryElementwiseOpOutputShape(inputs[0].Shape()); break; + case PrimitiveOpType::PackedIndex: + assert(inputs.size() == 2); + outputShape = UnaryElementwiseOpOutputShape(inputs[1].Shape()); + break; + case PrimitiveOpType::ScatterPacked: + { + assert(inputs.size() == 3); + if (inputs[0].DynamicAxes().empty() || inputs[1].DynamicAxes().empty() || inputs[2].DynamicAxes().empty()) + InvalidArgument("ScatterPacked requires all its operands to have dynamic axes"); + + outputShape = UnaryElementwiseOpOutputShape(inputs[0].Shape()); + break; } case PrimitiveOpType::TransposeAxes: { @@ -389,36 +458,6 @@ namespace CNTK assert(inputs.size() == 1); outputShape = {1}; break; - case PrimitiveOpType::Plus: - case PrimitiveOpType::LogPlus: - case PrimitiveOpType::Minus: - case PrimitiveOpType::ElementTimes: - case PrimitiveOpType::Equal: - case PrimitiveOpType::NotEqual: - case PrimitiveOpType::Less: - case PrimitiveOpType::LessEqual: - case PrimitiveOpType::Greater: - case PrimitiveOpType::GreaterEqual: - case PrimitiveOpType::PastValue: - case PrimitiveOpType::FutureValue: - { - assert(inputs.size() == 2); - if ((op == PrimitiveOpType::PastValue) || (op == PrimitiveOpType::FutureValue)) - { - Variable inputOperandVar = inputs[0]; - Variable initialStateVar = inputs[1]; - - // TODO: We currently only support input operand with 1 dynamic axis for PastValue/FutureValue - if ((inputOperandVar.DynamicAxes() != Axis::UnknownDynamicAxes()) && (inputOperandVar.DynamicAxes().size() != 2)) - LogicError("Currently PastValue/FutureValue Function only supports input operand with 2 dynamic axis (1 sequence-axis and 1 batch-axis)"); - - if (!initialStateVar.DynamicAxes().empty()) - LogicError("Currently PastValue/FutureValue Function does not support initial state operand with dynamic axes!"); - } - - outputShape = BinaryElementwiseOpOutputShape(op, inputs[0], inputs[1], true, inferDimensions); - break; - } case PrimitiveOpType::Times: { assert(inputs.size() == 2); @@ -523,9 +562,6 @@ namespace CNTK outputShape = BatchNormalizationOutputShape(inputs, spatial, inferDimensions); break; } - case PrimitiveOpType::PackedIndex: - outputShape = UnaryElementwiseOpOutputShape(inputs[1].Shape()); - break; case PrimitiveOpType::GatherPacked: { bool sourceHasDynamicAxis = !inputs[0].DynamicAxes().empty(); @@ -543,22 +579,6 @@ namespace CNTK break; } - case PrimitiveOpType::ScatterPacked: - { - if (inputs[0].DynamicAxes().empty() || inputs[1].DynamicAxes().empty() || inputs[2].DynamicAxes().empty()) - InvalidArgument("ScatterPacked requires all its operands to have dynamic axes"); - - outputShape = inputs[0].Shape(); - break; - } - case PrimitiveOpType::Clip: - assert(inputs.size() == 3); - outputShape = UnaryElementwiseOpOutputShape(inputs[0].Shape()); - break; - case PrimitiveOpType::Select: - assert(inputs.size() == 3); - outputShape = NaryElementwiseOpOutputShape(op, inputs, true); - break; case PrimitiveOpType::Splice: { assert(inputs.size() >= 2); @@ -645,6 +665,8 @@ namespace CNTK default: LogicError("Specified op %S not yet supported", PrimitiveOpTypeName(op).c_str()); break; + } + } } } diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.h b/Source/CNTKv2LibraryDll/PrimitiveFunction.h index 05ba9322684d..8cb1bb66544b 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.h +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.h @@ -421,6 +421,12 @@ namespace CNTK auto leftOperandShape = leftOperand.Shape(); auto rightOperandShape = rightOperand.Shape(); + if (leftOperandShape == NDShape::Unknown) + leftOperandShape = rightOperandShape; + + if (rightOperandShape == NDShape::Unknown) + rightOperandShape = leftOperandShape; + // All operand shapes should be known assert((leftOperandShape != NDShape::Unknown) && (rightOperandShape != NDShape::Unknown)); @@ -470,16 +476,16 @@ namespace CNTK return NDShape(std::move(outputDims)); } - static NDShape NaryElementwiseOpOutputShape(PrimitiveOpType op, std::vector& operands, bool broadcastAllowed) + static NDShape NaryElementwiseOpOutputShape(PrimitiveOpType op, std::vector& operands, bool broadcastAllowed, bool inferInputDimensions) { assert(operands.size() > 1); // TODO: Is this logic of transitively constructing the output shape from the operands correct? Variable dummyOutputVariable = PlaceholderVariable(NDShape()); for (auto& operand : operands) - dummyOutputVariable.m_dataFields->m_shape = BinaryElementwiseOpOutputShape(op, dummyOutputVariable, operand, broadcastAllowed, false); + dummyOutputVariable.m_dataFields->m_shape = BinaryElementwiseOpOutputShape(op, dummyOutputVariable, operand, broadcastAllowed, inferInputDimensions); - return dummyOutputVariable.m_dataFields->m_shape; + return dummyOutputVariable.Shape(); } // Returns a pair comprising of the output shape and boolean indicating if any input operand shape was modified @@ -684,6 +690,8 @@ namespace CNTK // TODO: Reconcile this with the ComputationNode::Validate functionality in core CNTK to avoid duplication of inference logic // Returns a pair of determined output variables and a bool indicating if any input operand shape was modified + static DataType GetOutputDataType(PrimitiveOpType op, std::vector& inputs, bool inferDimensions); + static std::vector GetOutputDynamicAxes(PrimitiveOpType op, std::vector& inputs, Dictionary& functionConfig); static std::vector GetOutputVariables(PrimitiveOpType op, std::vector& inputs, Dictionary& functionConfig, diff --git a/bindings/python/cntk/ops/tests/function_tests.py b/bindings/python/cntk/ops/tests/function_tests.py index e0f4b58a4edb..910d863c4d1e 100644 --- a/bindings/python/cntk/ops/tests/function_tests.py +++ b/bindings/python/cntk/ops/tests/function_tests.py @@ -13,7 +13,7 @@ from ..functions import * from ...trainer import * from ...initializer import glorot_uniform -from .. import constant, parameter, input_variable, placeholder_variable, times, plus +from .. import constant, parameter, input_variable, placeholder_variable, times, plus, past_value from ... import InferredDimension from .ops_test_utils import compare_lists_of_np_arrays @@ -155,3 +155,12 @@ def test_data_type_inference(): x_times_param1 = times(x_float, param1) assert (param1.dtype == np.float64) + +def test_recurrence_shape_inference(): + i = input_variable((2,)) + p = placeholder_variable() + p_past = past_value(p) + p_past_plus_i = p_past + i + + p_past_plus_i.replace_placeholder(p_past_plus_i.output) + assert p_past_plus_i.output.shape == (2,) From 487d1697785dab10bfd159ae160177b319c82e31 Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Wed, 11 Jan 2017 15:08:39 -0800 Subject: [PATCH 104/120] CNT v2 library: Bug fixes a) Duplicate argument mappings in Block arguments map when the block's inputs contain duplicate variables b) Verify MBlayout matches for input data specified for variables with same dynamic axes --- Source/CNTKv2LibraryDll/BlockFunction.h | 19 +- Source/CNTKv2LibraryDll/CompositeFunction.cpp | 28 +- Source/CNTKv2LibraryDll/CompositeFunction.h | 2 +- .../ComputeInputStatistics.cpp | 3 +- Source/CNTKv2LibraryDll/PrimitiveFunction.cpp | 702 +++++++++--------- bindings/python/cntk/ops/tests/block_test.py | 11 + .../python/cntk/ops/tests/function_tests.py | 19 +- 7 files changed, 411 insertions(+), 373 deletions(-) diff --git a/Source/CNTKv2LibraryDll/BlockFunction.h b/Source/CNTKv2LibraryDll/BlockFunction.h index 1cabc503f5d9..9ea559e29226 100644 --- a/Source/CNTKv2LibraryDll/BlockFunction.h +++ b/Source/CNTKv2LibraryDll/BlockFunction.h @@ -40,24 +40,25 @@ namespace CNTK // Mapping from each argument of the composite underlying the block to the corresponding Variable it is mapped to std::vector> CompositeArgumentsMap() const { - std::unordered_map argumentsMappingAsMap; + std::vector> argumentsMap; auto arguments = m_composite->Arguments(); for (auto argument : arguments) { if (argument.BlockFunctionVariableMapping() == Variable()) LogicError("BlockFunction (%S) with OpName (%S) does not have a mapping for argument (%S)", Name().c_str(), OpName().c_str(), argument.Name().c_str()); - argumentsMappingAsMap[argument] = argument.BlockFunctionVariableMapping(); + argumentsMap.push_back({ argument, argument.BlockFunctionVariableMapping() }); } - std::vector> argumentsMap; + // Now sort the mapping by the order of occurence of the argument mapping in the block's inputs auto blockInputs = Inputs(); - for (auto blockInput : blockInputs) - { - auto iter = std::find_if(argumentsMappingAsMap.begin(), argumentsMappingAsMap.end(), [&blockInput](const std::pair& entry) {return entry.second == blockInput; }); - if (iter != argumentsMappingAsMap.end()) - argumentsMap.push_back({iter->first, iter->second}); - } + std::unordered_map inputIndices; + for (size_t i = 0; i < blockInputs.size(); ++i) + inputIndices.insert({ blockInputs[i], i }); + + std::stable_sort(argumentsMap.begin(), argumentsMap.end(), [&inputIndices](const std::pair& first, const std::pair& second) { + return inputIndices.at(first.second) < inputIndices.at(second.second); + }); return argumentsMap; } diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.cpp b/Source/CNTKv2LibraryDll/CompositeFunction.cpp index f26c3bbd8d5e..75189c1ef683 100644 --- a/Source/CNTKv2LibraryDll/CompositeFunction.cpp +++ b/Source/CNTKv2LibraryDll/CompositeFunction.cpp @@ -1058,7 +1058,7 @@ namespace CNTK } template - /*static*/ void CompositeFunction::PopulateComputationNodeValue(const std::pair& variableValue, ComputationNodeBasePtr& computationNode) + /*static*/ void CompositeFunction::PopulateComputationNodeValue(const std::pair& variableValue, ComputationNodeBasePtr& computationNode, std::unordered_map& layoutsPopulated) { if (!computationNode->Is>()) LogicError("CompositeFunction::Forward: Illegal to populate value of computation node type other than InputValueBase!"); @@ -1070,17 +1070,27 @@ namespace CNTK else CNTKMatrixAndMBLayout = Utils::GetCNTKImplMatrixAndMBLayoutFromValueObject(variableValue.first, variableValue.second); - MBLayoutPtr layout = CNTKMatrixAndMBLayout.second; - - auto& nodeData = computationNode->As>()->Value(); - // Switch the node matrix to the right matrix type + auto& nodeData = computationNode->As>()->Value(); nodeData.AssignValuesOf(*CNTKMatrixAndMBLayout.first); - computationNode->GetMBLayout()->CopyFrom(layout); + + auto layout = CNTKMatrixAndMBLayout.second; + auto& nodeLayout = computationNode->GetMBLayout(); + if (layoutsPopulated.find(nodeLayout) == layoutsPopulated.end()) + { + nodeLayout->CopyFrom(layout); + layoutsPopulated.insert({ nodeLayout, variableValue.first }); + } + else + { + if (*nodeLayout != *layout) + InvalidArgument("Function::Forward: Different minibatch layouts detected (difference in sequence lengths or count or start flags) in data specified for 2 of the Function's argument ('%S', '%S') having same dynamic axes", variableValue.first.Name().c_str(), layoutsPopulated.at(nodeLayout).Name().c_str()); + } } void CompositeFunction::PopulateNetworkInputs(const std::unordered_map& arguments) { + std::unordered_map layoutsPopulated; std::vector inputNodes; for (auto argumentValuePair : arguments) { @@ -1090,15 +1100,13 @@ namespace CNTK inputNodes.push_back(argumentComputationNode); ValuePtr argumentValue = arguments.at(argument); - - MBLayoutPtr layout; switch (argumentValue->GetDataType()) { case DataType::Float: - PopulateComputationNodeValue({ argument, argumentValue }, argumentComputationNode); + PopulateComputationNodeValue({ argument, argumentValue }, argumentComputationNode, layoutsPopulated); break; case DataType::Double: - PopulateComputationNodeValue({ argument, argumentValue }, argumentComputationNode); + PopulateComputationNodeValue({ argument, argumentValue }, argumentComputationNode, layoutsPopulated); break; default: LogicError("Unsupported DataType %s", DataTypeName(argumentValue->GetDataType())); diff --git a/Source/CNTKv2LibraryDll/CompositeFunction.h b/Source/CNTKv2LibraryDll/CompositeFunction.h index 9a4e1ba9151e..326601d50ab0 100644 --- a/Source/CNTKv2LibraryDll/CompositeFunction.h +++ b/Source/CNTKv2LibraryDll/CompositeFunction.h @@ -254,7 +254,7 @@ namespace CNTK std::unordered_map& isVariableRootMap); template - static void PopulateComputationNodeValue(const std::pair& variableValue, Microsoft::MSR::CNTK::ComputationNodeBasePtr& computationNode); + static void PopulateComputationNodeValue(const std::pair& variableValue, Microsoft::MSR::CNTK::ComputationNodeBasePtr& computationNode, std::unordered_map< Microsoft::MSR::CNTK::MBLayoutPtr, Variable>& layoutsPopulated); void PopulateNetworkInputs(const std::unordered_map& arguments); template diff --git a/Source/CNTKv2LibraryDll/ComputeInputStatistics.cpp b/Source/CNTKv2LibraryDll/ComputeInputStatistics.cpp index 2f73a0f25896..17edda562558 100644 --- a/Source/CNTKv2LibraryDll/ComputeInputStatistics.cpp +++ b/Source/CNTKv2LibraryDll/ComputeInputStatistics.cpp @@ -74,6 +74,7 @@ namespace CNTK for (auto & preComputeNode : preComputeNodes) dynamic_pointer_cast(preComputeNode)->MarkComputed(false /*begin accumulating*/); + std::unordered_map layoutsPopulated; const size_t maxMinibatchDataSize = (1 << 27); // 128 MB const size_t minibatchSize = maxMinibatchDataSize / totalSizePerSample; for (;;) @@ -83,7 +84,7 @@ namespace CNTK break; for (auto& currentStreamKV : computedMeanAndInvStdDevs) - CompositeFunction::PopulateComputationNodeValue({ streamToDummyInputVariableMap[currentStreamKV.first], minibatchData[currentStreamKV.first].m_data }, streamToInputNodeMap[currentStreamKV.first]); + CompositeFunction::PopulateComputationNodeValue({ streamToDummyInputVariableMap[currentStreamKV.first], minibatchData[currentStreamKV.first].m_data }, streamToInputNodeMap[currentStreamKV.first], layoutsPopulated); ComputationNetwork::BumpEvalTimeStamp(allInputNodes); diff --git a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp index 418b6439d7c1..b0f50a6f375a 100644 --- a/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp +++ b/Source/CNTKv2LibraryDll/PrimitiveFunction.cpp @@ -221,7 +221,8 @@ namespace CNTK NDShape outputShape = NDShape::Unknown; bool allInputShapesUnknown = (std::find_if(inputs.begin(), inputs.end(), [](const Variable& input) { return !input.Shape().IsUnknown(); }) == inputs.end()); - if (!allInputShapesUnknown && (outputDynamicAxes != Axis::UnknownDynamicAxes())) + bool anyInputShapesUnknown = (std::find_if(inputs.begin(), inputs.end(), [](const Variable& input) { return input.Shape().IsUnknown(); }) != inputs.end()); + if (!anyInputShapesUnknown || (!allInputShapesUnknown && (outputDynamicAxes != Axis::UnknownDynamicAxes()))) { switch (op) { @@ -266,32 +267,31 @@ namespace CNTK break; default: // For all other operations, shapes of all inputs must be known to determine the output shape - bool anyInputShapesUnknown = (std::find_if(inputs.begin(), inputs.end(), [](const Variable& input) { return input.Shape().IsUnknown(); }) != inputs.end()); if (!anyInputShapesUnknown) - { - switch (op) - { - case PrimitiveOpType::Negate: - case PrimitiveOpType::Sigmoid: - case PrimitiveOpType::Tanh: - case PrimitiveOpType::ReLU: - case PrimitiveOpType::Exp: - case PrimitiveOpType::Log: - case PrimitiveOpType::Sqrt: - case PrimitiveOpType::Floor: - case PrimitiveOpType::Abs: - case PrimitiveOpType::Reciprocal: - case PrimitiveOpType::Softmax: - case PrimitiveOpType::Hardmax: - case PrimitiveOpType::Dropout: - case PrimitiveOpType::Where: - case PrimitiveOpType::LogSoftmax: - case PrimitiveOpType::Sin: - case PrimitiveOpType::Cos: - case PrimitiveOpType::Pass: - assert(inputs.size() == 1); - outputShape = UnaryElementwiseOpOutputShape(inputs[0].Shape()); - break; + { + switch (op) + { + case PrimitiveOpType::Negate: + case PrimitiveOpType::Sigmoid: + case PrimitiveOpType::Tanh: + case PrimitiveOpType::ReLU: + case PrimitiveOpType::Exp: + case PrimitiveOpType::Log: + case PrimitiveOpType::Sqrt: + case PrimitiveOpType::Floor: + case PrimitiveOpType::Abs: + case PrimitiveOpType::Reciprocal: + case PrimitiveOpType::Softmax: + case PrimitiveOpType::Hardmax: + case PrimitiveOpType::Dropout: + case PrimitiveOpType::Where: + case PrimitiveOpType::LogSoftmax: + case PrimitiveOpType::Sin: + case PrimitiveOpType::Cos: + case PrimitiveOpType::Pass: + assert(inputs.size() == 1); + outputShape = UnaryElementwiseOpOutputShape(inputs[0].Shape()); + break; case PrimitiveOpType::PackedIndex: assert(inputs.size() == 2); outputShape = UnaryElementwiseOpOutputShape(inputs[1].Shape()); @@ -304,367 +304,367 @@ namespace CNTK outputShape = UnaryElementwiseOpOutputShape(inputs[0].Shape()); break; - } - case PrimitiveOpType::TransposeAxes: - { - assert(inputs.size() == 1); + } + case PrimitiveOpType::TransposeAxes: + { + assert(inputs.size() == 1); - auto axis1 = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis1].Value(), inputs[0].Shape()); - auto axis2 = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis2].Value(), inputs[0].Shape()); + auto axis1 = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis1].Value(), inputs[0].Shape()); + auto axis2 = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis2].Value(), inputs[0].Shape()); - if (!axis1.IsStaticAxis() || !axis2.IsStaticAxis()) - LogicError("TransposeAxes operation currently does not support transposing dynamic axes"); + if (!axis1.IsStaticAxis() || !axis2.IsStaticAxis()) + LogicError("TransposeAxes operation currently does not support transposing dynamic axes"); - // We allow to transpose with an axes that exceeds the rank of the input. - // The output rank is the max of the input rank, and either of the axes being transposed. - auto outputRank = std::max(inputs[0].Shape().Rank(), (size_t)(std::max(axis1.StaticAxisIndex(), axis2.StaticAxisIndex()) + 1)); - outputShape = inputs[0].Shape().AppendShape(NDShape(outputRank - inputs[0].Shape().Rank(), 1)); - std::swap(outputShape[axis1.StaticAxisIndex()], outputShape[axis2.StaticAxisIndex()]); - break; - } - case PrimitiveOpType::Slice: - { - assert(inputs.size() == 1); - auto axis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis].Value(), inputs[0].Shape()); - - auto beginIndex = functionConfig[PrimitiveFunction::AttributeNameBeginIndex].Value(); - auto endIndex = functionConfig[PrimitiveFunction::AttributeNameEndIndex].Value(); - if (!axis.IsStaticAxis()) - LogicError("Built-in Slice operation currently does not support slicing along dynamic axis"); - - VerifyStaticAxis(axis, inputs[0].Shape()); - - size_t sliceAxisDim = inputs[0].Shape()[axis.StaticAxisIndex()]; - int realBeginIndex = (beginIndex >= 0) ? beginIndex : beginIndex + sliceAxisDim; - int realEndIndex = (endIndex > 0) ? endIndex : endIndex + sliceAxisDim; - if ((sliceAxisDim < realEndIndex) || (realEndIndex < realBeginIndex) || (realBeginIndex < 0)) - RuntimeError("Slice operation: Index range [%d,%d), interpreted as [%d,%d), is invalid for input's shape ([%S]).", - beginIndex, - endIndex, - realBeginIndex, - realEndIndex, - AsStringForErrorReporting(inputs[0].Shape()).c_str()); - - auto outputTensorShape = AsTensorShape(inputs[0].Shape()); - - // propagate as much as we can - if ((axis.StaticAxisIndex() < (int)outputTensorShape.GetRank()) && (0 <= realBeginIndex) && (realBeginIndex <= realEndIndex) && (realEndIndex <= sliceAxisDim)) - outputTensorShape.NarrowTo(axis.StaticAxisIndex(), realBeginIndex, realEndIndex); - - outputShape = AsNDShape(outputTensorShape, /*allowNonFlattenableTensorShapes = */ true); - break; - } - case PrimitiveOpType::Reshape: - { - auto& replacementShape = functionConfig[PrimitiveFunction::AttributeNameNewShape].Value(); + // We allow to transpose with an axes that exceeds the rank of the input. + // The output rank is the max of the input rank, and either of the axes being transposed. + auto outputRank = std::max(inputs[0].Shape().Rank(), (size_t)(std::max(axis1.StaticAxisIndex(), axis2.StaticAxisIndex()) + 1)); + outputShape = inputs[0].Shape().AppendShape(NDShape(outputRank - inputs[0].Shape().Rank(), 1)); + std::swap(outputShape[axis1.StaticAxisIndex()], outputShape[axis2.StaticAxisIndex()]); + break; + } + case PrimitiveOpType::Slice: + { + assert(inputs.size() == 1); + auto axis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis].Value(), inputs[0].Shape()); + + auto beginIndex = functionConfig[PrimitiveFunction::AttributeNameBeginIndex].Value(); + auto endIndex = functionConfig[PrimitiveFunction::AttributeNameEndIndex].Value(); + if (!axis.IsStaticAxis()) + LogicError("Built-in Slice operation currently does not support slicing along dynamic axis"); + + VerifyStaticAxis(axis, inputs[0].Shape()); + + size_t sliceAxisDim = inputs[0].Shape()[axis.StaticAxisIndex()]; + int realBeginIndex = (beginIndex >= 0) ? beginIndex : beginIndex + sliceAxisDim; + int realEndIndex = (endIndex > 0) ? endIndex : endIndex + sliceAxisDim; + if ((sliceAxisDim < realEndIndex) || (realEndIndex < realBeginIndex) || (realBeginIndex < 0)) + RuntimeError("Slice operation: Index range [%d,%d), interpreted as [%d,%d), is invalid for input's shape ([%S]).", + beginIndex, + endIndex, + realBeginIndex, + realEndIndex, + AsStringForErrorReporting(inputs[0].Shape()).c_str()); + + auto outputTensorShape = AsTensorShape(inputs[0].Shape()); + + // propagate as much as we can + if ((axis.StaticAxisIndex() < (int)outputTensorShape.GetRank()) && (0 <= realBeginIndex) && (realBeginIndex <= realEndIndex) && (realEndIndex <= sliceAxisDim)) + outputTensorShape.NarrowTo(axis.StaticAxisIndex(), realBeginIndex, realEndIndex); + + outputShape = AsNDShape(outputTensorShape, /*allowNonFlattenableTensorShapes = */ true); + break; + } + case PrimitiveOpType::Reshape: + { + auto& replacementShape = functionConfig[PrimitiveFunction::AttributeNameNewShape].Value(); - auto beginAxis = Axis(0); - auto endAxis = Axis((int)inputs[0].Shape().Rank()); - if (functionConfig.Contains(PrimitiveFunction::AttributeNameBeginAxis)) - beginAxis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameBeginAxis].Value(), inputs[0].Shape()); + auto beginAxis = Axis(0); + auto endAxis = Axis((int)inputs[0].Shape().Rank()); + if (functionConfig.Contains(PrimitiveFunction::AttributeNameBeginAxis)) + beginAxis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameBeginAxis].Value(), inputs[0].Shape()); - if (functionConfig.Contains(PrimitiveFunction::AttributeNameEndAxis)) - endAxis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameEndAxis].Value(), inputs[0].Shape()); + if (functionConfig.Contains(PrimitiveFunction::AttributeNameEndAxis)) + endAxis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameEndAxis].Value(), inputs[0].Shape()); - outputShape = ReshapeOutputShape(inputs[0].Shape(), replacementShape, beginAxis, endAxis, inferDimensions); - break; - } - case PrimitiveOpType::ROIPooling: - { - assert(inputs.size() == 2); - auto convMapShape = inputs[0].Shape(); - auto roisShape = inputs[1].Shape(); - auto roiOutputShape = functionConfig[PrimitiveFunction::AttributeNameROIOutputShape].Value(); + outputShape = ReshapeOutputShape(inputs[0].Shape(), replacementShape, beginAxis, endAxis, inferDimensions); + break; + } + case PrimitiveOpType::ROIPooling: + { + assert(inputs.size() == 2); + auto convMapShape = inputs[0].Shape(); + auto roisShape = inputs[1].Shape(); + auto roiOutputShape = functionConfig[PrimitiveFunction::AttributeNameROIOutputShape].Value(); - auto outW = roiOutputShape[0]; - auto outH = roiOutputShape[1]; - auto numChannels = convMapShape[2]; - auto roisPerImage = roisShape[1]; + auto outW = roiOutputShape[0]; + auto outH = roiOutputShape[1]; + auto numChannels = convMapShape[2]; + auto roisPerImage = roisShape[1]; - if (roiOutputShape.Rank() != 2) - InvalidArgument("ROIPoolingNode: roi output shape must have two dimensions ([W x H])."); + if (roiOutputShape.Rank() != 2) + InvalidArgument("ROIPoolingNode: roi output shape must have two dimensions ([W x H])."); - if (convMapShape[0] < outW || convMapShape[1] < outH) - InvalidArgument("ROIPoolingNode: inputWidth must >= windowWidth and inputHeight must >= windowHeight."); + if (convMapShape[0] < outW || convMapShape[1] < outH) + InvalidArgument("ROIPoolingNode: inputWidth must >= windowWidth and inputHeight must >= windowHeight."); - if (convMapShape[2] < 1) - InvalidArgument("ROIPoolingNode: input must have at least one channel ([W x H x C])."); + if (convMapShape[2] < 1) + InvalidArgument("ROIPoolingNode: input must have at least one channel ([W x H x C])."); - if (roisShape[0] != 4) - InvalidArgument("ROIPoolingNode: ROI input must have the following shape: [4 x roisPerImage]."); + if (roisShape[0] != 4) + InvalidArgument("ROIPoolingNode: ROI input must have the following shape: [4 x roisPerImage]."); - if (roisPerImage < 1) - InvalidArgument("ROIPoolingNode: ROI input must contain at least one ROI ([4 x roisPerImage])."); + if (roisPerImage < 1) + InvalidArgument("ROIPoolingNode: ROI input must contain at least one ROI ([4 x roisPerImage])."); - outputShape = { outW, outH, numChannels, roisPerImage }; - break; - } - case PrimitiveOpType::Pooling: - { - assert(inputs.size() == 1); - auto poolingWindowsShape = functionConfig[PrimitiveFunction::AttributeNamePoolingWindowShape].Value(); - auto strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); - auto lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); - auto upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); - auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); - NDShape outputMapCount = { 1 }; - std::vector sharing = { true }; - auto inputShape = inputs[0].Shape(); - - // In case of pooling if the kernel shape is unknown, then treat it as global pooling. - if (poolingWindowsShape == NDShape::Unknown) - { - if ((std::find(autoPadding.begin(), autoPadding.end(), true) != autoPadding.end()) || - (lowerPad.TotalSize() > 0) || (upperPad.TotalSize() > 0)) - RuntimeError("Padding isn't allowed for Unknown shape!"); + outputShape = { outW, outH, numChannels, roisPerImage }; + break; + } + case PrimitiveOpType::Pooling: + { + assert(inputs.size() == 1); + auto poolingWindowsShape = functionConfig[PrimitiveFunction::AttributeNamePoolingWindowShape].Value(); + auto strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); + auto lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); + auto upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); + auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); + NDShape outputMapCount = { 1 }; + std::vector sharing = { true }; + auto inputShape = inputs[0].Shape(); + + // In case of pooling if the kernel shape is unknown, then treat it as global pooling. + if (poolingWindowsShape == NDShape::Unknown) + { + if ((std::find(autoPadding.begin(), autoPadding.end(), true) != autoPadding.end()) || + (lowerPad.TotalSize() > 0) || (upperPad.TotalSize() > 0)) + RuntimeError("Padding isn't allowed for Unknown shape!"); - poolingWindowsShape = inputShape.SubShape(0, inputShape.Rank()-1); - functionConfig[PrimitiveFunction::AttributeNamePoolingWindowShape] = poolingWindowsShape; - } + poolingWindowsShape = inputShape.SubShape(0, inputShape.Rank() - 1); + functionConfig[PrimitiveFunction::AttributeNamePoolingWindowShape] = poolingWindowsShape; + } - outputShape = ConvolutionOpOutputShape(op, inputShape, poolingWindowsShape, outputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, false, inferDimensions); - break; - } - case PrimitiveOpType::Unpooling: - { - assert(inputs.size() == 2); + outputShape = ConvolutionOpOutputShape(op, inputShape, poolingWindowsShape, outputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, false, inferDimensions); + break; + } + case PrimitiveOpType::Unpooling: + { + assert(inputs.size() == 2); - auto inputShape = inputs[0].Shape(); - outputShape = inputs[1].Shape(); - PoolingType unpoolingType = (PoolingType)(functionConfig[PrimitiveFunction::AttributeNamePoolingType].Value()); - if (unpoolingType != PoolingType::Max) - LogicError("Only max unpooling is currently supported"); - - // Finding the shape of an unpooling operation from the input to be unpooled alone is ambiguous - // For example a 4x4 input with a 5x5 kernel a stride of 2x2 - // and padding could have resulted from pooling a 7x7 or 8x8 image - // Therefore what needs to happen here is to check whether the - // outputShape can be pooled into the inputShape using the specified attributes - auto unpoolingWindowShape = functionConfig[PrimitiveFunction::AttributeNameUnpoolingWindowShape].Value(); - auto strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); - auto lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); - auto upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); - auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); - NDShape inputMapCount = { 1 }; - std::vector sharing = { true }; - - NDShape inferredInputShape = ConvolutionOpOutputShape(PrimitiveOpType::Pooling, outputShape, unpoolingWindowShape, inputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, false, inferDimensions); - if (inferredInputShape != inputShape) - RuntimeError("The shape of the unpooling operand %ls is different from the result of pooling the poolingInput argument using the provided options %ls", inputShape.AsString().c_str(), inferredInputShape.AsString().c_str()); + auto inputShape = inputs[0].Shape(); + outputShape = inputs[1].Shape(); + PoolingType unpoolingType = (PoolingType)(functionConfig[PrimitiveFunction::AttributeNamePoolingType].Value()); + if (unpoolingType != PoolingType::Max) + LogicError("Only max unpooling is currently supported"); + + // Finding the shape of an unpooling operation from the input to be unpooled alone is ambiguous + // For example a 4x4 input with a 5x5 kernel a stride of 2x2 + // and padding could have resulted from pooling a 7x7 or 8x8 image + // Therefore what needs to happen here is to check whether the + // outputShape can be pooled into the inputShape using the specified attributes + auto unpoolingWindowShape = functionConfig[PrimitiveFunction::AttributeNameUnpoolingWindowShape].Value(); + auto strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); + auto lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); + auto upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); + auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); + NDShape inputMapCount = { 1 }; + std::vector sharing = { true }; + + NDShape inferredInputShape = ConvolutionOpOutputShape(PrimitiveOpType::Pooling, outputShape, unpoolingWindowShape, inputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, false, inferDimensions); + if (inferredInputShape != inputShape) + RuntimeError("The shape of the unpooling operand %ls is different from the result of pooling the poolingInput argument using the provided options %ls", inputShape.AsString().c_str(), inferredInputShape.AsString().c_str()); - break; - } - case PrimitiveOpType::SumAll: - assert(inputs.size() == 1); - outputShape = {1}; - break; - case PrimitiveOpType::Times: - { - assert(inputs.size() == 2); - auto outputRank = functionConfig[PrimitiveFunction::AttributeNameOutputRank].Value(); - auto inferInputRankToMap = functionConfig[PrimitiveFunction::AttributeNameInferInputRankToMap].Value(); - outputShape = TimesOpOutputShape(inputs[0], inputs[1], outputRank, inferInputRankToMap, inferDimensions); - break; - } - case PrimitiveOpType::TransposeTimes: - { - assert(inputs.size() == 2); + break; + } + case PrimitiveOpType::SumAll: + assert(inputs.size() == 1); + outputShape = { 1 }; + break; + case PrimitiveOpType::Times: + { + assert(inputs.size() == 2); + auto outputRank = functionConfig[PrimitiveFunction::AttributeNameOutputRank].Value(); + auto inferInputRankToMap = functionConfig[PrimitiveFunction::AttributeNameInferInputRankToMap].Value(); + outputShape = TimesOpOutputShape(inputs[0], inputs[1], outputRank, inferInputRankToMap, inferDimensions); + break; + } + case PrimitiveOpType::TransposeTimes: + { + assert(inputs.size() == 2); - auto transposeShapeFunc = [](const NDShape& shape) { - NDShape transposedShape(std::max(2, shape.Rank()), 1); - for (size_t i = 0; i < shape.Rank(); ++i) - transposedShape[transposedShape.Rank() - i - 1] = shape[i]; + auto transposeShapeFunc = [](const NDShape& shape) { + NDShape transposedShape(std::max(2, shape.Rank()), 1); + for (size_t i = 0; i < shape.Rank(); ++i) + transposedShape[transposedShape.Rank() - i - 1] = shape[i]; - return transposedShape; - }; + return transposedShape; + }; - if (inputs[0].Shape().Rank() > 2) - LogicError("TransposeTimes operation currently only supports %s operands of rank 1 or 2", Internal::IsReversingTensorShapesInErrorMessagesEnabled() ? "right" : "left"); + if (inputs[0].Shape().Rank() > 2) + LogicError("TransposeTimes operation currently only supports %s operands of rank 1 or 2", Internal::IsReversingTensorShapesInErrorMessagesEnabled() ? "right" : "left"); - NDShape transposedLeftOperandShape = transposeShapeFunc(inputs[0].Shape()); - Variable dummyLeftOperand = PlaceholderVariable(transposedLeftOperandShape); - size_t outputRank = functionConfig[PrimitiveFunction::AttributeNameOutputRank].Value(); - outputShape = TimesOpOutputShape(dummyLeftOperand, inputs[1], outputRank, -1, inferDimensions); - if (dummyLeftOperand.Shape() != transposedLeftOperandShape) - inputs[0].m_dataFields->m_shape = transposeShapeFunc(dummyLeftOperand.Shape()); + NDShape transposedLeftOperandShape = transposeShapeFunc(inputs[0].Shape()); + Variable dummyLeftOperand = PlaceholderVariable(transposedLeftOperandShape); + size_t outputRank = functionConfig[PrimitiveFunction::AttributeNameOutputRank].Value(); + outputShape = TimesOpOutputShape(dummyLeftOperand, inputs[1], outputRank, -1, inferDimensions); + if (dummyLeftOperand.Shape() != transposedLeftOperandShape) + inputs[0].m_dataFields->m_shape = transposeShapeFunc(dummyLeftOperand.Shape()); - break; - } - case PrimitiveOpType::Convolution: - { - assert(inputs.size() == 2); - auto& strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); - auto& lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); - auto& upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); - auto sharing = AsVector(functionConfig[PrimitiveFunction::AttributeNameSharing].Value>()); - auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); - bool transpose = functionConfig[PrimitiveFunction::AttributeNameTranspose].Value(); - if (inputs[0].Shape().Rank() < inputs[1].Shape().Rank()) - InvalidArgument("The convolution map should have at least as many axes as the shape of the input it operates on!"); - - NDShape outputMapCount, kernelShape; - std::tie(outputMapCount, kernelShape) = GetConvolutionOutputMapCountAndKernelShape(inputs[0].Shape(), inputs[1].Shape()); - auto originalKernelShape = kernelShape; - outputShape = ConvolutionOpOutputShape(op, inputs[1].Shape(), kernelShape, outputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, transpose, inferDimensions); - if (originalKernelShape != kernelShape) - { - for (size_t i2 = 0; i2 < kernelShape.Rank(); ++i2) - inputs[0].m_dataFields->m_shape[i2] = kernelShape[i2]; - } + break; + } + case PrimitiveOpType::Convolution: + { + assert(inputs.size() == 2); + auto& strides = functionConfig[PrimitiveFunction::AttributeNameStrides].Value(); + auto& lowerPad = functionConfig[PrimitiveFunction::AttributeNameLowerPad].Value(); + auto& upperPad = functionConfig[PrimitiveFunction::AttributeNameUpperPad].Value(); + auto sharing = AsVector(functionConfig[PrimitiveFunction::AttributeNameSharing].Value>()); + auto autoPadding = AsVector(functionConfig[PrimitiveFunction::AttributeNameAutoPadding].Value>()); + bool transpose = functionConfig[PrimitiveFunction::AttributeNameTranspose].Value(); + if (inputs[0].Shape().Rank() < inputs[1].Shape().Rank()) + InvalidArgument("The convolution map should have at least as many axes as the shape of the input it operates on!"); + + NDShape outputMapCount, kernelShape; + std::tie(outputMapCount, kernelShape) = GetConvolutionOutputMapCountAndKernelShape(inputs[0].Shape(), inputs[1].Shape()); + auto originalKernelShape = kernelShape; + outputShape = ConvolutionOpOutputShape(op, inputs[1].Shape(), kernelShape, outputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, transpose, inferDimensions); + if (originalKernelShape != kernelShape) + { + for (size_t i2 = 0; i2 < kernelShape.Rank(); ++i2) + inputs[0].m_dataFields->m_shape[i2] = kernelShape[i2]; + } - functionConfig[PrimitiveFunction::AttributeNameSharing] = AsDictionaryValueVector(sharing); - functionConfig[PrimitiveFunction::AttributeNameAutoPadding] = AsDictionaryValueVector(autoPadding); - break; - } - case PrimitiveOpType::CosDistance: - case PrimitiveOpType::Logistic: - case PrimitiveOpType::SquaredError: - case PrimitiveOpType::CrossEntropyWithSoftmax: - case PrimitiveOpType::ClassificationError: - { - if ((op == PrimitiveOpType::ClassificationError) || (op == PrimitiveOpType::Logistic)) - assert(inputs.size() >= 2); - else - assert(inputs.size() == 2); + functionConfig[PrimitiveFunction::AttributeNameSharing] = AsDictionaryValueVector(sharing); + functionConfig[PrimitiveFunction::AttributeNameAutoPadding] = AsDictionaryValueVector(autoPadding); + break; + } + case PrimitiveOpType::CosDistance: + case PrimitiveOpType::Logistic: + case PrimitiveOpType::SquaredError: + case PrimitiveOpType::CrossEntropyWithSoftmax: + case PrimitiveOpType::ClassificationError: + { + if ((op == PrimitiveOpType::ClassificationError) || (op == PrimitiveOpType::Logistic)) + assert(inputs.size() >= 2); + else + assert(inputs.size() == 2); - if ((inputs[0].Shape().Rank() > 2) || ((inputs[0].Shape().Rank() > 1) && (inputs[0].Shape()[1] != 1))) - InvalidArgument("The shape of input operands for the %S operation should have at most one axis", PrimitiveOpTypeName(op).c_str()); + if ((inputs[0].Shape().Rank() > 2) || ((inputs[0].Shape().Rank() > 1) && (inputs[0].Shape()[1] != 1))) + InvalidArgument("The shape of input operands for the %S operation should have at most one axis", PrimitiveOpTypeName(op).c_str()); - auto predictionShape = inputs[0].Shape(); - auto labelsShape = inputs[1].Shape(); - if (predictionShape != labelsShape) - RuntimeError("Prediction output operand's shape %S is incompatible with label operand's shape %S for the %S operation", AsStringForErrorReporting(predictionShape).c_str(), AsStringForErrorReporting(labelsShape).c_str(), PrimitiveOpTypeName(op).c_str()); + auto predictionShape = inputs[0].Shape(); + auto labelsShape = inputs[1].Shape(); + if (predictionShape != labelsShape) + RuntimeError("Prediction output operand's shape %S is incompatible with label operand's shape %S for the %S operation", AsStringForErrorReporting(predictionShape).c_str(), AsStringForErrorReporting(labelsShape).c_str(), PrimitiveOpTypeName(op).c_str()); - std::vector reductionAxes; - for (int i3 = 0; i3 < (int)inputs[0].Shape().Rank(); ++i3) - reductionAxes.push_back(i3); + std::vector reductionAxes; + for (int i3 = 0; i3 < (int)inputs[0].Shape().Rank(); ++i3) + reductionAxes.push_back(i3); - outputShape = ReductionOpOutputShape(op, predictionShape, reductionAxes, /*preserveReductionAxes =*/ false); - break; - } - case PrimitiveOpType::ReduceElements: - { - assert(inputs.size() == 1); - auto reductionAxis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis].Value(), inputs[0].Shape()); - if (reductionAxis == Axis::AllStaticAxes()) - outputShape = {}; - else - { - std::vector reductionAxes = { reductionAxis.StaticAxisIndex() }; - outputShape = ReductionOpOutputShape(op, inputs[0].Shape(), reductionAxes, /*preserveReductionAxes =*/ true); - } - break; - } - case PrimitiveOpType::BatchNormalization: - { - assert(inputs.size() == 5); - auto spatial = functionConfig[PrimitiveFunction::AttributeNameSpatial].Value(); - outputShape = BatchNormalizationOutputShape(inputs, spatial, inferDimensions); - break; - } - case PrimitiveOpType::GatherPacked: - { - bool sourceHasDynamicAxis = !inputs[0].DynamicAxes().empty(); + outputShape = ReductionOpOutputShape(op, predictionShape, reductionAxes, /*preserveReductionAxes =*/ false); + break; + } + case PrimitiveOpType::ReduceElements: + { + assert(inputs.size() == 1); + auto reductionAxis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis].Value(), inputs[0].Shape()); + if (reductionAxis == Axis::AllStaticAxes()) + outputShape = {}; + else + { + std::vector reductionAxes = { reductionAxis.StaticAxisIndex() }; + outputShape = ReductionOpOutputShape(op, inputs[0].Shape(), reductionAxes, /*preserveReductionAxes =*/ true); + } + break; + } + case PrimitiveOpType::BatchNormalization: + { + assert(inputs.size() == 5); + auto spatial = functionConfig[PrimitiveFunction::AttributeNameSpatial].Value(); + outputShape = BatchNormalizationOutputShape(inputs, spatial, inferDimensions); + break; + } + case PrimitiveOpType::GatherPacked: + { + bool sourceHasDynamicAxis = !inputs[0].DynamicAxes().empty(); - // inherit tensor dimension from sourceData, minus the last (column or time) dimension. TODO this needs to become simpler... - if (sourceHasDynamicAxis) - outputShape = inputs[0].Shape(); - else - { - if (inputs[0].Shape().Rank() > 1) - outputShape = outputShape.SubShape(0, outputShape.Rank() - 1); - else - outputShape = {}; - } + // inherit tensor dimension from sourceData, minus the last (column or time) dimension. TODO this needs to become simpler... + if (sourceHasDynamicAxis) + outputShape = inputs[0].Shape(); + else + { + if (inputs[0].Shape().Rank() > 1) + outputShape = outputShape.SubShape(0, outputShape.Rank() - 1); + else + outputShape = {}; + } - break; - } - case PrimitiveOpType::Splice: - { - assert(inputs.size() >= 2); - auto maxInputRank = MaxInputRank(inputs); - auto spliceAxis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis].Value(), NDShape(maxInputRank)); + break; + } + case PrimitiveOpType::Splice: + { + assert(inputs.size() >= 2); + auto maxInputRank = MaxInputRank(inputs); + auto spliceAxis = NormalizeStaticAxis(functionConfig[PrimitiveFunction::AttributeNameAxis].Value(), NDShape(maxInputRank)); - if (!spliceAxis.IsStaticAxis()) - LogicError("Splice operation currently does not support splicing along dynamic axis"); + if (!spliceAxis.IsStaticAxis()) + LogicError("Splice operation currently does not support splicing along dynamic axis"); - if (spliceAxis.StaticAxisIndex() < 0) - InvalidArgument("Splice: The axis argument's static axis index must be >= 0!"); + if (spliceAxis.StaticAxisIndex() < 0) + InvalidArgument("Splice: The axis argument's static axis index must be >= 0!"); - outputShape = SpliceOutputShape(inputs, spliceAxis.StaticAxisIndex()); - break; - } - case PrimitiveOpType::RandomSample: - case PrimitiveOpType::RandomSampleInclusionFrequency: - { - auto numSamples = functionConfig[PrimitiveFunction::AttributeNameNumSamples].Value(); - auto allowDuplicates = functionConfig[PrimitiveFunction::AttributeNameAllowDuplicates].Value(); + outputShape = SpliceOutputShape(inputs, spliceAxis.StaticAxisIndex()); + break; + } + case PrimitiveOpType::RandomSample: + case PrimitiveOpType::RandomSampleInclusionFrequency: + { + auto numSamples = functionConfig[PrimitiveFunction::AttributeNameNumSamples].Value(); + auto allowDuplicates = functionConfig[PrimitiveFunction::AttributeNameAllowDuplicates].Value(); - if (numSamples == 0) - InvalidArgument("Number of requested samples is zero."); + if (numSamples == 0) + InvalidArgument("Number of requested samples is zero."); - let& shape = inputs[0].Shape(); - size_t numClasses = shape.Dimensions()[0]; + let& shape = inputs[0].Shape(); + size_t numClasses = shape.Dimensions()[0]; - if (numClasses != NDShape::InferredDimension && !allowDuplicates && numClasses <= numSamples) - InvalidArgument("For sampling without duplicates the number of requested samples (%lu) needs to be less than the number of classes (%lu).", numSamples, numClasses); - - // within this block we handle RandomSample and RandomSampleInclusionFrequency - if (op == PrimitiveOpType::RandomSampleInclusionFrequency) - outputShape = shape; - else - { - vector dimensions{ numClasses, numSamples}; - outputShape = NDShape(dimensions); - } + if (numClasses != NDShape::InferredDimension && !allowDuplicates && numClasses <= numSamples) + InvalidArgument("For sampling without duplicates the number of requested samples (%lu) needs to be less than the number of classes (%lu).", numSamples, numClasses); - break; - } - case PrimitiveOpType::OptimizedRNNStack: - { - assert(inputs.size() == 2); - auto operand = inputs[0]; - auto parameter = inputs[1]; - if (operand.Shape().Rank() != 1) - InvalidArgument("OptimizedRNNStack: input must have rank 1; actual input rank is %lu", operand.Shape().Rank()); - if (operand.DynamicAxes().empty()) - InvalidArgument("OptimizedRNNStack: input must have at least one dynamic axis"); - auto numLayers = functionConfig[PrimitiveFunction::AttributeNameNumLayers].Value(); - if (numLayers == 0) - InvalidArgument("Number of layers in OptimizedRNNStack operation should be positive"); - auto bidirectional = functionConfig[PrimitiveFunction::AttributeNameBidirectional].Value(); - auto hiddenSize = functionConfig[PrimitiveFunction::AttributeNameHiddenSize].Value(); - - // output dims - outputShape = operand.Shape(); - outputShape[0] = (bidirectional ? 2 : 1) * hiddenSize; - // infer input size - // Note: Output dim is second axis, so say initOutputRank=-1. - if (parameter.Shape().Rank() == 2) - { - const auto recurrentOp = functionConfig[PrimitiveFunction::AttributeNameRecurrentOp].Value(); - const auto attributes = RnnAttributes(bidirectional, numLayers, hiddenSize, recurrentOp, -1); - const auto numParameters = attributes.GetNumParameters(operand.Shape().TotalSize()); - std::vector> newOperandShapes = { { parameter, std::move(NDShape({ numParameters.first, numParameters.second })) } }; - UpdateOperandShapes(newOperandShapes); - } - break; - } - case PrimitiveOpType::ReconcileDynamicAxis: - { - assert(inputs.size() == 2); - auto operand = inputs[0]; - auto layout = inputs[1]; - if (operand.DynamicAxes().empty()) - InvalidArgument("ReconcileDynamicAxis: input must have at least one dynamic axis"); - if (layout.DynamicAxes().empty()) - InvalidArgument("ReconcileDynamicAxis: layout must have at least one dynamic axis"); - outputShape = operand.Shape(); - break; - } - default: - LogicError("Specified op %S not yet supported", PrimitiveOpTypeName(op).c_str()); - break; + // within this block we handle RandomSample and RandomSampleInclusionFrequency + if (op == PrimitiveOpType::RandomSampleInclusionFrequency) + outputShape = shape; + else + { + vector dimensions{ numClasses, numSamples }; + outputShape = NDShape(dimensions); + } + + break; + } + case PrimitiveOpType::OptimizedRNNStack: + { + assert(inputs.size() == 2); + auto operand = inputs[0]; + auto parameter = inputs[1]; + if (operand.Shape().Rank() != 1) + InvalidArgument("OptimizedRNNStack: input must have rank 1; actual input rank is %lu", operand.Shape().Rank()); + if (operand.DynamicAxes().empty()) + InvalidArgument("OptimizedRNNStack: input must have at least one dynamic axis"); + auto numLayers = functionConfig[PrimitiveFunction::AttributeNameNumLayers].Value(); + if (numLayers == 0) + InvalidArgument("Number of layers in OptimizedRNNStack operation should be positive"); + auto bidirectional = functionConfig[PrimitiveFunction::AttributeNameBidirectional].Value(); + auto hiddenSize = functionConfig[PrimitiveFunction::AttributeNameHiddenSize].Value(); + + // output dims + outputShape = operand.Shape(); + outputShape[0] = (bidirectional ? 2 : 1) * hiddenSize; + // infer input size + // Note: Output dim is second axis, so say initOutputRank=-1. + if (parameter.Shape().Rank() == 2) + { + const auto recurrentOp = functionConfig[PrimitiveFunction::AttributeNameRecurrentOp].Value(); + const auto attributes = RnnAttributes(bidirectional, numLayers, hiddenSize, recurrentOp, -1); + const auto numParameters = attributes.GetNumParameters(operand.Shape().TotalSize()); + std::vector> newOperandShapes = { { parameter, std::move(NDShape({ numParameters.first, numParameters.second })) } }; + UpdateOperandShapes(newOperandShapes); + } + break; + } + case PrimitiveOpType::ReconcileDynamicAxis: + { + assert(inputs.size() == 2); + auto operand = inputs[0]; + auto layout = inputs[1]; + if (operand.DynamicAxes().empty()) + InvalidArgument("ReconcileDynamicAxis: input must have at least one dynamic axis"); + if (layout.DynamicAxes().empty()) + InvalidArgument("ReconcileDynamicAxis: layout must have at least one dynamic axis"); + outputShape = operand.Shape(); + break; + } + default: + LogicError("Specified op %S not yet supported", PrimitiveOpTypeName(op).c_str()); + break; } } } diff --git a/bindings/python/cntk/ops/tests/block_test.py b/bindings/python/cntk/ops/tests/block_test.py index 10af400d32e5..b67795483494 100644 --- a/bindings/python/cntk/ops/tests/block_test.py +++ b/bindings/python/cntk/ops/tests/block_test.py @@ -99,3 +99,14 @@ def test_combine_op_as_block(): expected_forward = [[[2.]]] assert np.array_equal(res, expected_forward) + + +def test_block_with_duplicate_inputs(): + from .. import placeholder_variable, as_block, input_variable + input = input_variable((1,), name='input') + + left_operand_placeholder = placeholder_variable(name='left_placeholder') + right_operand_placeholder = placeholder_variable() + plus_block = as_block(right_operand_placeholder + left_operand_placeholder, [(left_operand_placeholder, input), (right_operand_placeholder, input)], 'plus') + + plus_block_clone = plus_block.clone('share') \ No newline at end of file diff --git a/bindings/python/cntk/ops/tests/function_tests.py b/bindings/python/cntk/ops/tests/function_tests.py index 910d863c4d1e..cb1ba1a567f0 100644 --- a/bindings/python/cntk/ops/tests/function_tests.py +++ b/bindings/python/cntk/ops/tests/function_tests.py @@ -13,7 +13,7 @@ from ..functions import * from ...trainer import * from ...initializer import glorot_uniform -from .. import constant, parameter, input_variable, placeholder_variable, times, plus, past_value +from .. import constant, parameter, input_variable, placeholder_variable, times, plus, past_value, sequence from ... import InferredDimension from .ops_test_utils import compare_lists_of_np_arrays @@ -164,3 +164,20 @@ def test_recurrence_shape_inference(): p_past_plus_i.replace_placeholder(p_past_plus_i.output) assert p_past_plus_i.output.shape == (2,) + +def test_sequence_data_mismatch(): + x = input_variable((1,), name='x') + ones = input_variable((1,), name='ones') + y_broadcast_last = sequence.broadcast_as(sequence.last(ones), x) + y_broadcast_first = sequence.broadcast_as(sequence.first(ones), x) + + x0 = np.array([1,2,3,4],dtype=np.float32).reshape(4,1) + o0 = np.array([1], dtype=np.float32).reshape(1,1) + + with pytest.raises(ValueError): + y_broadcast_last_result = y_broadcast_last.eval({x:[x0], ones:[o0]}) + + with pytest.raises(ValueError): + y_broadcast_first_result = y_broadcast_first.eval({x:[x0], ones:[o0]}) + + \ No newline at end of file From 35bcd6365c489e907e2e09a629581f5acc84ee14 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Thu, 12 Jan 2017 10:19:42 +0100 Subject: [PATCH 105/120] upgrade the solution file for eval samples to VS2015, remove EvalV2clients (it will be in another solution for V2 --- .../Evaluation/CSEvalClient/CSEvalClient.csproj | 15 ++++++++------- Examples/Evaluation/CSEvalClient/packages.config | 2 +- Examples/Evaluation/EvalClients.sln | 9 ++------- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj b/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj index 12bfc522a527..de7aca73a4d1 100644 --- a/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj +++ b/Examples/Evaluation/CSEvalClient/CSEvalClient.csproj @@ -11,7 +11,8 @@ CSEvalClient v4.5 512 - 931e68df + + publish\ true Disk @@ -36,7 +37,7 @@ full x64 prompt - MinimumRecommendedRules.ruleset + MinimumRecommendedRules.ruleset $(SolutionDir)..\..\$(Platform)\CSEvalClient.$(Configuration)\ @@ -45,10 +46,10 @@ pdbonly x64 prompt - MinimumRecommendedRules.ruleset + MinimumRecommendedRules.ruleset - + ..\packages\Microsoft.Research.CNTK.CpuEval-mkl.2.0-beta7\lib\net45\x64\EvalWrapper.dll True @@ -88,8 +89,8 @@ - This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. - - + + \ No newline at end of file diff --git a/Examples/Evaluation/CSEvalClient/packages.config b/Examples/Evaluation/CSEvalClient/packages.config index f9c25c49d226..8857df23853b 100644 --- a/Examples/Evaluation/CSEvalClient/packages.config +++ b/Examples/Evaluation/CSEvalClient/packages.config @@ -1,4 +1,4 @@  - + \ No newline at end of file diff --git a/Examples/Evaluation/EvalClients.sln b/Examples/Evaluation/EvalClients.sln index 87f3d5b818cc..01838fa76c53 100644 --- a/Examples/Evaluation/EvalClients.sln +++ b/Examples/Evaluation/EvalClients.sln @@ -1,14 +1,12 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 2013 -VisualStudioVersion = 12.0.40629.0 +# Visual Studio 14 +VisualStudioVersion = 14.0.25420.1 MinimumVisualStudioVersion = 10.0.40219.1 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CPPEvalClient", "CPPEvalClient\CPPEvalClient.vcxproj", "{C81CE839-184C-42C7-BB1C-9D0ABA17078D}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CSEvalClient", "CSEvalClient\CSEvalClient.csproj", "{92CCF4B9-BFED-4914-901A-CF1327B1A02D}" EndProject -Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CPPEvalV2Client", "CPPEvalV2Client\CPPEvalV2Client.vcxproj", "{D771A06D-CC25-4582-B5CD-D2A4782BB005}" -EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "CPPEvalExtendedClient", "CPPEvalExtendedClient\CPPEvalExtendedClient.vcxproj", "{93ECB70B-FDDD-44B4-BD6A-D63E094C704B}" EndProject Global @@ -24,9 +22,6 @@ Global {92CCF4B9-BFED-4914-901A-CF1327B1A02D}.Debug|x64.Build.0 = Debug|x64 {92CCF4B9-BFED-4914-901A-CF1327B1A02D}.Release|x64.ActiveCfg = Release|x64 {92CCF4B9-BFED-4914-901A-CF1327B1A02D}.Release|x64.Build.0 = Release|x64 - {D771A06D-CC25-4582-B5CD-D2A4782BB005}.Debug|x64.ActiveCfg = Release|x64 - {D771A06D-CC25-4582-B5CD-D2A4782BB005}.Release|x64.ActiveCfg = Release|x64 - {D771A06D-CC25-4582-B5CD-D2A4782BB005}.Release|x64.Build.0 = Release|x64 {93ECB70B-FDDD-44B4-BD6A-D63E094C704B}.Debug|x64.ActiveCfg = Release|x64 {93ECB70B-FDDD-44B4-BD6A-D63E094C704B}.Release|x64.ActiveCfg = Release|x64 {93ECB70B-FDDD-44B4-BD6A-D63E094C704B}.Release|x64.Build.0 = Release|x64 From 93b2133dc4f9c0569a8253ed4119b27f9abfac44 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Thu, 12 Jan 2017 11:15:16 +0100 Subject: [PATCH 106/120] change Readme file too. --- Examples/Evaluation/README.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Examples/Evaluation/README.md b/Examples/Evaluation/README.md index b155da3d4f65..667668bf78a1 100644 --- a/Examples/Evaluation/README.md +++ b/Examples/Evaluation/README.md @@ -2,12 +2,16 @@ The folder contains some examples using the CNTK evaluation library. Please note that only the 64-bit target is supported by CNTK evaluation library. --CPPEvalClient: demonstrate the use of the C++ CNTK eval lib. Only the release configuration is supported. +-CPPEvalClient: demonstrate the use of the C++ CNTK EvalDll evaluation API. Only the release configuration is supported. --CSEvalClient: demonstrate the use of the C# CNTK eval lib. +-CPPEvalExtendedClient: demonstrate the use of the C++ EvalDll EvalExtended API to evaluate a LSTM model. Only the release configuration is supported. --EvalClients.sln: the VS2013 solution file to build examples. It creates two binaries in the directory $(SolutionDir)..\..\x64\: +-CSEvalClient: demonstrate the use of the C# CNTK EvalDll Nuget Package. - - CPPEvalClient.$(Configuration)\CPPEvalClient.exe: the C++ example executable. To run the example, please first include the directory containing CNTK dependent dlls, usually $(SolutionDir)..\..\cntk, in the PATH environment variable. +-EvalClients.sln: the VS2015 solution file to build examples. It creates two binaries in the directory $(SolutionDir)..\..\x64\: + + - CPPEvalClient.$(Configuration)\CPPEvalClient.exe: To run the example, please first include the directory containing CNTK dependent dlls, usually $(SolutionDir)..\..\cntk, in the PATH environment variable. + + - CPPEvalExtendedClient.$(Configuration)\CPPEvalExtendedClient.exe: To run the example, please first include the directory containing CNTK dependent dlls, usually $(SolutionDir)..\..\cntk, in the PATH environment variable. - CSEvalClient.$(Configuration)\CSEvalClient.exe: the C# example executable. From 2a4c7f8031cd33e273ee93e96d44273ad03fed84 Mon Sep 17 00:00:00 2001 From: Zhou Wang Date: Thu, 12 Jan 2017 12:31:06 +0100 Subject: [PATCH 107/120] fix tabs and add a wiki link --- Examples/Evaluation/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Examples/Evaluation/README.md b/Examples/Evaluation/README.md index 667668bf78a1..feaa2bfb352f 100644 --- a/Examples/Evaluation/README.md +++ b/Examples/Evaluation/README.md @@ -1,6 +1,7 @@ #EvalClients The folder contains some examples using the CNTK evaluation library. Please note that only the 64-bit target is supported by CNTK evaluation library. +The [CNTK Eval Examples](https://github.com/Microsoft/CNTK/wiki/CNTK-Eval-Examples) page provides more details of these examples. -CPPEvalClient: demonstrate the use of the C++ CNTK EvalDll evaluation API. Only the release configuration is supported. @@ -11,7 +12,7 @@ The folder contains some examples using the CNTK evaluation library. Please note -EvalClients.sln: the VS2015 solution file to build examples. It creates two binaries in the directory $(SolutionDir)..\..\x64\: - CPPEvalClient.$(Configuration)\CPPEvalClient.exe: To run the example, please first include the directory containing CNTK dependent dlls, usually $(SolutionDir)..\..\cntk, in the PATH environment variable. - - - CPPEvalExtendedClient.$(Configuration)\CPPEvalExtendedClient.exe: To run the example, please first include the directory containing CNTK dependent dlls, usually $(SolutionDir)..\..\cntk, in the PATH environment variable. + + - CPPEvalExtendedClient.$(Configuration)\CPPEvalExtendedClient.exe: To run the example, please first include the directory containing CNTK dependent dlls, usually $(SolutionDir)..\..\cntk, in the PATH environment variable. - CSEvalClient.$(Configuration)\CSEvalClient.exe: the C# example executable. From 73f8319b01050e830521cdd163337e5455d1b60b Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 12 Jan 2017 13:47:09 +0100 Subject: [PATCH 108/120] Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py: fix env variable name --- .../CNTKv2Python/Examples/feature_extraction_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py b/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py index 6794c8646947..62ff3e1880dc 100644 --- a/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/feature_extraction_test.py @@ -24,7 +24,7 @@ def test_feature_extraction(device_id): set_default_device(cntk_device(device_id)) base_path = os.path.dirname(os.path.abspath(__file__)) - externalData = 'cCNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ + externalData = 'CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY' in os.environ if externalData: extPath = os.environ['CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY'] print("Reading data and model from %s" % extPath) From 0389f772d077141cf3c949d373a86ffe03aa3a97 Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Thu, 12 Jan 2017 13:34:23 +0100 Subject: [PATCH 109/120] fix the Anaconda installer invocation (no quotes allowed around path) clean up DoProcess function --- Tools/devInstall/Windows/helper/Action.ps1 | 26 +++++++------------ .../devInstall/Windows/helper/Operations.ps1 | 3 ++- 2 files changed, 12 insertions(+), 17 deletions(-) diff --git a/Tools/devInstall/Windows/helper/Action.ps1 b/Tools/devInstall/Windows/helper/Action.ps1 index c981e8bd45ab..a6beba187724 100644 --- a/Tools/devInstall/Windows/helper/Action.ps1 +++ b/Tools/devInstall/Windows/helper/Action.ps1 @@ -165,10 +165,12 @@ function InstallMSI( $completeMsiName = Join-Path $dir $msi - $cmd = "c:\Windows\System32\MSIEXEC.EXE" + $windDir = GetEnvironmentVariableContent("WINDIR") + $cmd = Join-Path $windDir "System32\MSIEXEC.EXE" + $param= "/i `"$completeMsiName`" /quiet /norestart" - DoProcess -doExecute $Execute -command $cmd -param "$param" -requiresRunAs $true -maxErrorLevel 0 -throwOnError $true + DoProcess -doExecute $Execute -command $cmd -param $param -requiresRunAs $true -maxErrorLevel 0 -throwOnError $true } function MakeDirectory( @@ -562,22 +564,14 @@ function DoProcess( return } - if ($workingDir.Length -eq 0) { - if ($requiresRunAs) { - $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -Verb runas - } - else { - $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru - } - + if (-not $workingDir) { + $workingDir = $(get-location).Path + } + if ($requiresRunAs) { + $process = start-process -FilePath $command -ArgumentList $param -Wait -PassThru -Verb runas -WorkingDirectory $workingDir } else { - if ($requiresRunAs) { - $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -Verb runas -WorkingDirectory "$workingDir" - } - else { - $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -WorkingDirectory "$workingDir" - } + $process = start-process -FilePath $command -ArgumentList $param -Wait -PassThru -WorkingDirectory $workingDir } $eCode = ($process.ExitCode) diff --git a/Tools/devInstall/Windows/helper/Operations.ps1 b/Tools/devInstall/Windows/helper/Operations.ps1 index ecbdca3239cd..ee7e507b1aae 100644 --- a/Tools/devInstall/Windows/helper/Operations.ps1 +++ b/Tools/devInstall/Windows/helper/Operations.ps1 @@ -17,7 +17,8 @@ function OpAnaconda3411( @( @{ShortName = "ANA3-411"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath; } ); Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); - Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/InstallationType=JustMe /AddToPath=0 /RegisterPython=0 /S /D=`"$targetPath`""; runAs=$false; Message = ".... This will take some time. Please be patient ...." } ); + # command line parameters for Anaconda installer: $targetPath must be the last parameter and can not be surrounded by quotes + Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/InstallationType=JustMe /AddToPath=0 /RegisterPython=0 /S /D=$targetPath"; runAs=$false; Message = ".... This will take some time. Please be patient ...." } ); } ) } From ec475cf5f3b1b1b805b3094103072ee3897f441a Mon Sep 17 00:00:00 2001 From: Wolfgang Manousek Date: Thu, 12 Jan 2017 13:58:07 +0100 Subject: [PATCH 110/120] adressed CR comments --- Tools/devInstall/Windows/helper/Action.ps1 | 2 +- Tools/devInstall/Windows/helper/Operations.ps1 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/devInstall/Windows/helper/Action.ps1 b/Tools/devInstall/Windows/helper/Action.ps1 index a6beba187724..223185e766ad 100644 --- a/Tools/devInstall/Windows/helper/Action.ps1 +++ b/Tools/devInstall/Windows/helper/Action.ps1 @@ -166,7 +166,7 @@ function InstallMSI( $completeMsiName = Join-Path $dir $msi $windDir = GetEnvironmentVariableContent("WINDIR") - $cmd = Join-Path $windDir "System32\MSIEXEC.EXE" + $cmd = Join-Path $windDir System32\MSIEXEC.EXE $param= "/i `"$completeMsiName`" /quiet /norestart" diff --git a/Tools/devInstall/Windows/helper/Operations.ps1 b/Tools/devInstall/Windows/helper/Operations.ps1 index ee7e507b1aae..ff7d8087203f 100644 --- a/Tools/devInstall/Windows/helper/Operations.ps1 +++ b/Tools/devInstall/Windows/helper/Operations.ps1 @@ -17,7 +17,7 @@ function OpAnaconda3411( @( @{ShortName = "ANA3-411"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath; } ); Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); - # command line parameters for Anaconda installer: $targetPath must be the last parameter and can not be surrounded by quotes + # command line parameters for Anaconda installer: /D=$targetPath must be the last parameter and can not be surrounded by quotes Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/InstallationType=JustMe /AddToPath=0 /RegisterPython=0 /S /D=$targetPath"; runAs=$false; Message = ".... This will take some time. Please be patient ...." } ); } ) } From 4665583810d486e0c5c3221e2963988614df7158 Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Tue, 10 Jan 2017 10:39:24 +0100 Subject: [PATCH 111/120] Make Jupyter Notebook tests device-aware --- .../CNTK_202_Language_Understanding_test.py | 2 ++ .../CNTKv2Python/Examples/conftest.py | 9 +++++++- Tutorials/CNTK_101_LogisticRegression.ipynb | 10 +++++++-- Tutorials/CNTK_102_FeedForward.ipynb | 22 +++++++------------ ...e_Timeseries_Basic_with_Pandas_Numpy.ipynb | 8 +++++++ .../CNTK_202_Language_Understanding.ipynb | 9 +++++++- Tutorials/CNTK_204_Sequence_To_Sequence.ipynb | 9 +++++++- .../CNTK_205_Artistic_Style_Transfer.ipynb | 8 ++++++- 8 files changed, 57 insertions(+), 20 deletions(-) diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_202_Language_Understanding_test.py b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_202_Language_Understanding_test.py index 72d3345bc34b..9f9833f45788 100644 --- a/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_202_Language_Understanding_test.py +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_202_Language_Understanding_test.py @@ -10,6 +10,8 @@ abs_path = os.path.dirname(os.path.abspath(__file__)) notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_202_Language_Understanding.ipynb") +# Runs on GPU only, batch normalization training on CPU is not yet implemented. +notebook_deviceIdsToRun = [0] def test_cntk_202_language_understanding_noErrors(nb): errors = [output for cell in nb.cells if 'outputs' in cell diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py b/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py index 3885591dfaa2..4e2801b6f116 100644 --- a/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py @@ -62,12 +62,19 @@ def nb(tmpdir_factory, request, device_id): import subprocess from cntk.ops.tests.ops_test_utils import cntk_device from cntk.cntk_py import DeviceKind_GPU + inPath = getattr(request.module, "notebook") + deviceIdsToRun = [-1, 0] + try: + deviceIdsToRun = getattr(request.module, "notebook_deviceIdsToRun") + except AttributeError: + pass # Pass along device_id type to child process if cntk_device(device_id).type() == DeviceKind_GPU: os.environ['TEST_DEVICE'] = 'gpu' else: os.environ['TEST_DEVICE'] = 'cpu' - inPath = getattr(request.module, "notebook") + if not device_id in deviceIdsToRun: + pytest.skip('test not configured to run on device ID {0}'.format(device_id)) outPath = str(tmpdir_factory.mktemp('notebook').join('out.ipynb')) assert os.path.isfile(inPath) kernel_name_opt = "--ExecutePreprocessor.kernel_name=python%d" % (sys.version_info[0]) diff --git a/Tutorials/CNTK_101_LogisticRegression.ipynb b/Tutorials/CNTK_101_LogisticRegression.ipynb index 68237e391d29..88bc92720a97 100644 --- a/Tutorials/CNTK_101_LogisticRegression.ipynb +++ b/Tutorials/CNTK_101_LogisticRegression.ipynb @@ -72,9 +72,15 @@ "import sys\n", "import os\n", "from cntk import Trainer, learning_rate_schedule, UnitType\n", - "from cntk.device import cpu, set_default_device\n", "from cntk.learner import sgd\n", - "from cntk.ops import *" + "from cntk.ops import *\n", + "# Select the right target device when this notebook is being tested:\n", + "if 'TEST_DEVICE' in os.environ:\n", + " import cntk\n", + " if os.environ['TEST_DEVICE'] == 'cpu':\n", + " cntk.device.set_default_device(cntk.device.cpu())\n", + " else:\n", + " cntk.device.set_default_device(cntk.device.gpu(0))\n" ] }, { diff --git a/Tutorials/CNTK_102_FeedForward.ipynb b/Tutorials/CNTK_102_FeedForward.ipynb index 9fff855216a7..9f4b257f0a96 100644 --- a/Tutorials/CNTK_102_FeedForward.ipynb +++ b/Tutorials/CNTK_102_FeedForward.ipynb @@ -66,20 +66,14 @@ "from cntk import Trainer, learning_rate_schedule, UnitType\n", "from cntk.device import cpu, set_default_device\n", "from cntk.learner import sgd\n", - "from cntk.ops import *" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "# Specify the target device to be used for computing (this example is showing for CPU usage)\n", - "# Note we need to set the device only once during the session.\n", - "set_default_device(cpu())" + "from cntk.ops import *\n", + "# Select the right target device when this notebook is being tested:\n", + "if 'TEST_DEVICE' in os.environ:\n", + " import cntk\n", + " if os.environ['TEST_DEVICE'] == 'cpu':\n", + " cntk.device.set_default_device(cntk.device.cpu())\n", + " else:\n", + " cntk.device.set_default_device(cntk.device.gpu(0))\n" ] }, { diff --git a/Tutorials/CNTK_104_Finance_Timeseries_Basic_with_Pandas_Numpy.ipynb b/Tutorials/CNTK_104_Finance_Timeseries_Basic_with_Pandas_Numpy.ipynb index 14811d3ced38..d5f2c15ca76f 100644 --- a/Tutorials/CNTK_104_Finance_Timeseries_Basic_with_Pandas_Numpy.ipynb +++ b/Tutorials/CNTK_104_Finance_Timeseries_Basic_with_Pandas_Numpy.ipynb @@ -25,6 +25,7 @@ "outputs": [], "source": [ "from __future__ import print_function\n", + "import os\n", "import numpy as np\n", "import cntk\n", "import cntk.ops as C\n", @@ -40,6 +41,13 @@ "pd.options.mode.chained_assignment = None # default='warn'\n", "%matplotlib inline\n", "\n", + "# Select the right target device when this notebook is being tested:\n", + "if 'TEST_DEVICE' in os.environ:\n", + " if os.environ['TEST_DEVICE'] == 'cpu':\n", + " cntk.device.set_default_device(cntk.device.cpu())\n", + " else:\n", + " cntk.device.set_default_device(cntk.device.gpu(0))\n", + "\n", "# If you want to set the program to use a CUDA enabled GPU\n", "#from cntk.device import set_default_device, gpu\n", "#set_default_device(gpu(0))" diff --git a/Tutorials/CNTK_202_Language_Understanding.ipynb b/Tutorials/CNTK_202_Language_Understanding.ipynb index aa9dc74f9947..da00e29a7f63 100644 --- a/Tutorials/CNTK_202_Language_Understanding.ipynb +++ b/Tutorials/CNTK_202_Language_Understanding.ipynb @@ -119,7 +119,14 @@ "from cntk.io import MinibatchSource, CTFDeserializer\n", "from cntk.io import StreamDef, StreamDefs, INFINITELY_REPEAT, FULL_DATA_SWEEP\n", "from cntk import *\n", - "from cntk.learner import adam_sgd, learning_rate_schedule" + "from cntk.learner import adam_sgd, learning_rate_schedule\n", + "# Select the right target device when this notebook is being tested:\n", + "if 'TEST_DEVICE' in os.environ:\n", + " import cntk\n", + " if os.environ['TEST_DEVICE'] == 'cpu':\n", + " cntk.device.set_default_device(cntk.device.cpu())\n", + " else:\n", + " cntk.device.set_default_device(cntk.device.gpu(0))\n" ] }, { diff --git a/Tutorials/CNTK_204_Sequence_To_Sequence.ipynb b/Tutorials/CNTK_204_Sequence_To_Sequence.ipynb index dfbc282fafed..55f1ad85f8c2 100644 --- a/Tutorials/CNTK_204_Sequence_To_Sequence.ipynb +++ b/Tutorials/CNTK_204_Sequence_To_Sequence.ipynb @@ -104,7 +104,14 @@ "from cntk.ops.functions import CloneMethod\n", "from cntk.blocks import LSTM, Stabilizer\n", "from cntk.initializer import glorot_uniform\n", - "from cntk.utils import get_train_eval_criterion, get_train_loss" + "from cntk.utils import get_train_eval_criterion, get_train_loss\n", + "# Select the right target device when this notebook is being tested:\n", + "if 'TEST_DEVICE' in os.environ:\n", + " import cntk\n", + " if os.environ['TEST_DEVICE'] == 'cpu':\n", + " cntk.device.set_default_device(cntk.device.cpu())\n", + " else:\n", + " cntk.device.set_default_device(cntk.device.gpu(0))\n" ] }, { diff --git a/Tutorials/CNTK_205_Artistic_Style_Transfer.ipynb b/Tutorials/CNTK_205_Artistic_Style_Transfer.ipynb index a7083e9c2bc3..640eeb9204a7 100644 --- a/Tutorials/CNTK_205_Artistic_Style_Transfer.ipynb +++ b/Tutorials/CNTK_205_Artistic_Style_Transfer.ipynb @@ -32,7 +32,13 @@ "import h5py\n", "import os\n", "%matplotlib inline\n", - "import matplotlib.pyplot as plt" + "import matplotlib.pyplot as plt\n", + "# Select the right target device when this notebook is being tested:\n", + "if 'TEST_DEVICE' in os.environ:\n", + " if os.environ['TEST_DEVICE'] == 'cpu':\n", + " C.device.set_default_device(C.device.cpu())\n", + " else:\n", + " C.device.set_default_device(C.device.gpu(0))\n" ] }, { From cb66befcf27af3fd9d52c4c209e87e0973b3e78c Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 12 Jan 2017 13:27:08 +0100 Subject: [PATCH 112/120] CNTK_203_Reinforcement_Learning_Basics.ipynb: add test --- ...CNTK_103B_MNIST_FeedForwardNetwork_test.py | 2 - ..._203_Reinforcement_Learning_Basics_test.py | 21 ++++++ .../CNTKv2Python/Examples/conftest.py | 11 ++- ...TK_203_Reinforcement_Learning_Basics.ipynb | 74 ++++++++++++++----- 4 files changed, 87 insertions(+), 21 deletions(-) create mode 100644 Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py index 518c974e58e4..bdecad260d09 100644 --- a/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_103B_MNIST_FeedForwardNetwork_test.py @@ -19,8 +19,6 @@ def test_cntk_103_mnist_feedforwardnetwork_noErrors(nb): expectedEvalErrorByDeviceId = { -1: 1.90, 0: 1.85 } def test_cntk_103_mnist_feedforwardnetwork_evalCorrect(nb, device_id): - testCell = [cell for cell in nb.cells - if cell.cell_type == 'code' and cell.source.find("print(\"Average test error:") != -1] testCell = [cell for cell in nb.cells if cell.cell_type == 'code' and re.search('trainer\.test_minibatch', cell.source)] assert len(testCell) == 1 diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py new file mode 100644 index 000000000000..cb62638c76c8 --- /dev/null +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py @@ -0,0 +1,21 @@ +# Copyright (c) Microsoft. All rights reserved. + +# Licensed under the MIT license. See LICENSE.md file in the project root +# for full license information. +# ============================================================================== + +import os +import re + +abs_path = os.path.dirname(os.path.abspath(__file__)) +notebook = os.path.join(abs_path, "..", "..", "..", "..", "Tutorials", "CNTK_203_Reinforcement_Learning_Basics.ipynb") + +def test_cntk_203_reinforcement_learning_basics_noErrors(nb): + errors = [output for cell in nb.cells if 'outputs' in cell + for output in cell['outputs'] if output.output_type == "error"] + print(errors) + assert errors == [] + +def test_cntk_203_reinforcement_learning_basics_tasks_are_solved(nb): + testCells = [cell for cell in nb.cells if cell.cell_type == 'code' and len(cell.outputs) > 0 and 'text' in cell.outputs[0] and re.search('Task solved in[ :]', cell.outputs[0]['text'])] + assert len(testCells) == 2 diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py b/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py index 4e2801b6f116..f0bf6c6279ac 100644 --- a/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/conftest.py @@ -63,11 +63,19 @@ def nb(tmpdir_factory, request, device_id): from cntk.ops.tests.ops_test_utils import cntk_device from cntk.cntk_py import DeviceKind_GPU inPath = getattr(request.module, "notebook") + deviceIdsToRun = [-1, 0] try: deviceIdsToRun = getattr(request.module, "notebook_deviceIdsToRun") except AttributeError: pass + + timeoutSeconds = 300 + try: + timeoutSeconds = int(getattr(request.module, "notebook_timeoutSeconds")) + except AttributeError: + pass + # Pass along device_id type to child process if cntk_device(device_id).type() == DeviceKind_GPU: os.environ['TEST_DEVICE'] = 'gpu' @@ -79,7 +87,8 @@ def nb(tmpdir_factory, request, device_id): assert os.path.isfile(inPath) kernel_name_opt = "--ExecutePreprocessor.kernel_name=python%d" % (sys.version_info[0]) args = ["jupyter", "nbconvert", "--to", "notebook", "--execute", - "--ExecutePreprocessor.timeout=300", kernel_name_opt, "--output", outPath, inPath] + "--ExecutePreprocessor.timeout={0}".format(timeoutSeconds), + kernel_name_opt, "--output", outPath, inPath] subprocess.check_call(args) nb = nbformat.read(outPath, nbformat.current_nbformat) return nb diff --git a/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb b/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb index a24262dee0e2..bb9e61508d35 100644 --- a/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb +++ b/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb @@ -59,6 +59,7 @@ "outputs": [], "source": [ "from __future__ import print_function\n", + "from __future__ import division\n", "import matplotlib.pyplot as plt\n", "from matplotlib import style\n", "import numpy as np\n", @@ -91,6 +92,29 @@ " import gym" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Select the notebook run mode\n", + "\n", + "There are two run modes:\n", + "- *Fast mode*: `isFast` is set to `True`. This is the default mode for the notebooks, which means we train for fewer iterations or train / test on limited data. This ensures functional correctness of the notebook though the models produced are far from what a completed training would produce.\n", + "\n", + "- *Slow mode*: We recommend the user to set this flag to `False` once the user has gained familiarity with the notebook content and wants to gain insight from running the notebooks for a longer period with different parameters for training. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "isFast = True" + ] + }, { "cell_type": "markdown", "metadata": { @@ -118,7 +142,9 @@ " * the cart is moving more than 2.4 units from center.\n", " \n", "The task is considered done, if\n", - " * the agent achieved and averaged reward of 200 over the last 50 episodes (if you manage to get a reward of 200 averaged over the last 100 episode you can consider submitting it to OpenAI)" + " * the agent achieved and averaged reward of 200 over the last 50 episodes (if you manage to get a reward of 200 averaged over the last 100 episode you can consider submitting it to OpenAI).\n", + "\n", + "In fast mode these targets are relaxed." ] }, { @@ -172,7 +198,12 @@ "#from keras.optimizers import *\n", "from cntk import *\n", "from cntk.models import Sequential\n", - "from cntk.layers import *" + "from cntk.layers import *# Select the right target device when this notebook is being tested:\n", + "if 'TEST_DEVICE' in os.environ:\n", + " if os.environ['TEST_DEVICE'] == 'cpu':\n", + " device.set_default_device(device.cpu())\n", + " else:\n", + " device.set_default_device(device.gpu(0))\n" ] }, { @@ -219,7 +250,11 @@ }, "outputs": [], "source": [ - "BATCH_SIZE_BASELINE = 50 # calculate average reward over these many episodes\n", + "# Targetted reward\n", + "REWARD_TARGET = 30 if isFast else 200\n", + "# Averaged over these these many episodes\n", + "BATCH_SIZE_BASELINE = 20 if isFast else 50\n", + "\n", "H = 64 # hidden layer size\n", "\n", "class Brain:\n", @@ -262,7 +297,7 @@ "\n", " def train(self, x, y, epoch=1, verbose=0):\n", " #self.model.fit(x, y, batch_size=64, nb_epoch=epoch, verbose=verbose)\n", - " arguments = dict(zip(self.loss.arguments, [y,x]))\n", + " arguments = dict(zip(self.loss.arguments, [x,y]))\n", " updated, results =self.trainer.train_minibatch(arguments, outputs=[self.loss.output])\n", "\n", " def predict(self, s):\n", @@ -453,6 +488,8 @@ }, "outputs": [], "source": [ + "TOTAL_EPISODES = 1500 if isFast else 3000\n", + "\n", "def run(agent):\n", " s = env.reset()\n", " R = 0 \n", @@ -481,7 +518,7 @@ "\n", "episode_number = 0\n", "reward_sum = 0\n", - "while episode_number<3000:\n", + "while episode_number < TOTAL_EPISODES:\n", " reward_sum += run(agent)\n", " episode_number += 1\n", " if episode_number % BATCH_SIZE_BASELINE == 0:\n", @@ -489,7 +526,7 @@ " reward_sum / BATCH_SIZE_BASELINE))\n", " if episode_number%200==0:\n", " plot_weights([(agent.brain.params['W1'], 'Episode %i $W_1$'%episode_number)], figsize=(14,5))\n", - " if reward_sum / BATCH_SIZE_BASELINE > 200:\n", + " if reward_sum / BATCH_SIZE_BASELINE > REWARD_TARGET:\n", " print('Task solved in %d episodes' % episode_number)\n", " plot_weights([(agent.brain.params['W1'], 'Episode %i $W_1$'%episode_number)], figsize=(14,5)) \n", " break\n", @@ -502,16 +539,16 @@ "metadata": {}, "source": [ "If you run it, you should see something like\n", - "\n", - "```[2016-10-26 22:06:25,436] Making new env: CartPole-v0\n", - "Episode: 50, Average reward for episode 23.700000.\n", - "Episode: 100, Average reward for episode 18.720000.\n", - "Episode: 150, Average reward for episode 17.960000.\n", + "```\n", + "Episode: 20, Average reward for episode 20.700000.\n", + "Episode: 40, Average reward for episode 20.150000.\n", + "Episode: 60, Average reward for episode 21.100000.\n", "...\n", - "Episode: 1750, Average reward for episode 100.180000.\n", - "Episode: 1800, Average reward for episode 111.380000.\n", - "Episode: 1850, Average reward for episode 207.240000.\n", - "Task solved in 1850 episodes```" + "Episode: 960, Average reward for episode 20.150000.\n", + "Episode: 980, Average reward for episode 26.700000.\n", + "Episode: 1000, Average reward for episode 32.900000.\n", + "Task solved in 1000 episodes\n", + "```" ] }, { @@ -551,7 +588,7 @@ "for i_episode in range(num_episodes):\n", " print(i_episode)\n", " while not done:\n", - " env.render()\n", + " #env.render()\n", " action = np.argmax(root.eval([observation.astype(np.float32)]))\n", " observation, reward, done, info = env.step(action)\n", " if done:\n", @@ -678,7 +715,8 @@ "source": [ "import cntk as C \n", "\n", - "TOTAL_EPISODES = 10000\n", + "TOTAL_EPISODES = 1500 if isFast else 10000\n", + "\n", "D = 4 # input dimensionality\n", "H = 10 # number of hidden layer neurons\n", "\n", @@ -786,7 +824,7 @@ "\n", " print('Episode: %d. Average reward for episode %f.' % (episode_number, reward_sum / BATCH_SIZE_BASELINE))\n", "\n", - " if reward_sum / BATCH_SIZE_BASELINE > 200:\n", + " if reward_sum / BATCH_SIZE_BASELINE > REWARD_TARGET:\n", " print('Task solved in: %d ' % episode_number)\n", " break\n", "\n", From 48d79a351804cd700e147b8af2a0f6a3a56beedf Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 12 Jan 2017 13:59:24 +0100 Subject: [PATCH 113/120] CNTK_203_Reinforcement_Learning_Basics.ipynb: just spacing --- ...TK_203_Reinforcement_Learning_Basics.ipynb | 74 +++++++++---------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb b/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb index bb9e61508d35..0e5de472d8ac 100644 --- a/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb +++ b/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb @@ -7,7 +7,7 @@ "# CNTK 203: Reinforcement Learning Basics\n", "\n", "\n", - "Reinforcement learning (RL) is an area of machine learning inspired by behaviorist psychology, concerned with how [software agents](https://en.wikipedia.org/wiki/Software_agent) ought to take [actions](https://en.wikipedia.org/wiki/Action_selection) in an environment so as to maximize some notion of cumulative reward. In machine learning, the environment is typically formulated as a [Markov decision process](https://en.wikipedia.org/wiki/Markov_decision_process) (MDP) as many reinforcement learning algorithms for this context utilize [dynamic programming](https://en.wikipedia.org/wiki/Dynamic_programming) techniques. \n", + "Reinforcement learning (RL) is an area of machine learning inspired by behaviorist psychology, concerned with how [software agents](https://en.wikipedia.org/wiki/Software_agent) ought to take [actions](https://en.wikipedia.org/wiki/Action_selection) in an environment so as to maximize some notion of cumulative reward. In machine learning, the environment is typically formulated as a [Markov decision process](https://en.wikipedia.org/wiki/Markov_decision_process) (MDP) as many reinforcement learning algorithms for this context utilize [dynamic programming](https://en.wikipedia.org/wiki/Dynamic_programming) techniques.\n", "\n", "In some machine learning settings, we do not have immediate access to labels, so we cannot rely on supervised learning techniques. If, however, there is something we can interact with and thereby get some feedback that tells us occasionally, whether our previous behavior was good or not, we can use RL to learn how to improve our behavior.\n", "\n", @@ -20,16 +20,16 @@ "We will use the [CartPole](https://gym.openai.com/envs/CartPole-v0) environment from OpenAI's [gym](https://github.com/openai/gym) simulator to teach a cart to balance a pole. As described in the link above, in the CartPole example, a pole is attached by an un-actuated joint to a cart, which moves along a frictionless track. The system is controlled by applying a force of +1 or -1 to the cart. A reward of +1 is provided for every timestep that the pole remains upright. The episode ends when the pole is more than 15 degrees from vertical, or the cart moves more than 2.4 units from the center. See figure below for reference.\n", "\n", "**Goal**\n", - "Our goal is to prevent the pole from falling over as the cart moves with the pole in upright position (perpendicular to the cart) as the starting state. More specifically if the pole is less than 15 degrees from vertical while the cart is within 2.4 units of the center we will collect reward. In this tutorial, we will train till we learn a set of actions (policies) that lead to an average reward of 200 or more over last 50 batches. \n", + "Our goal is to prevent the pole from falling over as the cart moves with the pole in upright position (perpendicular to the cart) as the starting state. More specifically if the pole is less than 15 degrees from vertical while the cart is within 2.4 units of the center we will collect reward. In this tutorial, we will train till we learn a set of actions (policies) that lead to an average reward of 200 or more over last 50 batches.\n", "\n", - "In, RL terminology, the goal is to find _policies_ $a$, that maximize the _reward_ $r$ (feedback) through interaction with some environment (in this case the pole being balanced on the cart). So given a series of experiences $$s \\xrightarrow{a} r, s'$$ we then can learn how to choose action $a$ in a given state $s$ to maximize the accumulated reward $r$ over time: \n", + "In, RL terminology, the goal is to find _policies_ $a$, that maximize the _reward_ $r$ (feedback) through interaction with some environment (in this case the pole being balanced on the cart). So given a series of experiences $$s \\xrightarrow{a} r, s'$$ we then can learn how to choose action $a$ in a given state $s$ to maximize the accumulated reward $r$ over time:\n", "\\begin{align}\n", "Q(s,a) &= r_0 + \\gamma r_1 + \\gamma^2 r_2 + \\ldots \\newline\n", "&= r_0 + \\gamma \\max_a Q^*(s',a)\n", "\\end{align}\n", - "where $\\gamma \\in [0,1)$ is the discount factor that controls how much we should value reward that is further away. This is called the [*Bellmann*-equation](https://en.wikipedia.org/wiki/Bellman_equation). \n", + "where $\\gamma \\in [0,1)$ is the discount factor that controls how much we should value reward that is further away. This is called the [*Bellmann*-equation](https://en.wikipedia.org/wiki/Bellman_equation).\n", "\n", - "In this tutorial we will show how to model the state space, how to use the received reward to figure out which action yields the highest future reward. \n", + "In this tutorial we will show how to model the state space, how to use the received reward to figure out which action yields the highest future reward.\n", "\n", "We present two different popular approaches here:\n", "\n", @@ -37,7 +37,7 @@ "\n", "**Policy gradient**: This method directly estimates the policy (set of actions) in the network. The outcome is a learning of an ordered set of actions which leads to maximize reward by probabilistically choosing a subset of actions. In this tutorial, we learn the actions using a gradient descent approach to learn the policies.\n", "\n", - "In this tutorial, we focus how to implement RL in CNTK. We choose a straight forward shallow network. One can extend the approaches by replacing our shallow model with deeper networks that are introduced in other CNTK tutorials. \n", + "In this tutorial, we focus how to implement RL in CNTK. We choose a straight forward shallow network. One can extend the approaches by replacing our shallow model with deeper networks that are introduced in other CNTK tutorials.\n", "\n", "Additionally, this tutorial is in its early stages and will be evolving in future updates." ] @@ -133,14 +133,14 @@ "In every time step, the agent\n", " * gets an observation $(x, \\dot{x}, \\theta, \\dot{\\theta})$, corresponding to *cart position*, *cart velocity*, *pole angle with the vertical*, *pole angular velocity*,\n", " * performs an action `LEFT` or `RIGHT`, and\n", - " * receives \n", + " * receives\n", " * a reward of +1 for having survived another time step, and\n", " * a new state $(x', \\dot{x}', \\theta', \\dot{\\theta}')$\n", - " \n", - "The episode ends, if \n", + "\n", + "The episode ends, if\n", " * the pole is more than 15 degrees from vertical and/or\n", " * the cart is moving more than 2.4 units from center.\n", - " \n", + "\n", "The task is considered done, if\n", " * the agent achieved and averaged reward of 200 over the last 50 episodes (if you manage to get a reward of 200 averaged over the last 100 episode you can consider submitting it to OpenAI).\n", "\n", @@ -176,7 +176,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "We will start with a slightly modified version for Keras, https://github.com/jaara/AI-blog/blob/master/CartPole-basic.py, published by Jaromír Janisch in his [AI blog](https://jaromiru.com/2016/09/27/lets-make-a-dqn-theory/), and will then incrementally convert it to use CNTK. \n", + "We will start with a slightly modified version for Keras, https://github.com/jaara/AI-blog/blob/master/CartPole-basic.py, published by Jaromír Janisch in his [AI blog](https://jaromiru.com/2016/09/27/lets-make-a-dqn-theory/), and will then incrementally convert it to use CNTK.\n", "\n", "We use a simple two-layer densely connected network, for simpler illustrations. More advance networks can be substituted.\n", "\n", @@ -262,7 +262,7 @@ " self.params = {}\n", " self.model, self.trainer, self.loss = self._create()\n", " # self.model.load_weights(\"cartpole-basic.h5\")\n", - " \n", + "\n", " def _create(self):\n", " observation = input_variable(STATE_COUNT, np.float32, name=\"s\")\n", " q_target = input_variable(ACTION_COUNT, np.float32, name=\"q\")\n", @@ -271,14 +271,14 @@ " # model.add(Dense(output_dim=64, activation='relu', input_dim=STATE_COUNT))\n", " # model.add(Dense(output_dim=ACTION_COUNT, activation='linear'))\n", "\n", - " # Following a style similar to Keras \n", + " # Following a style similar to Keras\n", " l1 = Dense(H, activation=relu)\n", " l2 = Dense(ACTION_COUNT)\n", " unbound_model = Sequential([l1, l2])\n", " model = unbound_model(observation)\n", "\n", - " self.params = dict(W1=l1.W, b1=l1.b, W2=l2.W, b2=l2.b) \n", - " \n", + " self.params = dict(W1=l1.W, b1=l1.b, W2=l2.W, b2=l2.b)\n", + "\n", " lr = 0.00025\n", " # opt = RMSprop(lr=0.00025)\n", " # model.compile(loss='mse', optimizer=opt)\n", @@ -326,7 +326,7 @@ " self.capacity = capacity\n", "\n", " def add(self, sample):\n", - " self.samples.append(sample) \n", + " self.samples.append(sample)\n", "\n", " if len(self.samples) > self.capacity:\n", " self.samples.pop(0)\n", @@ -367,7 +367,7 @@ " def __init__(self):\n", " self.brain = Brain()\n", " self.memory = Memory(MEMORY_CAPACITY)\n", - " \n", + "\n", " def act(self, s):\n", " if random.random() < self.epsilon:\n", " return random.randint(0, ACTION_COUNT-1)\n", @@ -375,19 +375,19 @@ " return numpy.argmax(self.brain.predict(s))\n", "\n", " def observe(self, sample): # in (s, a, r, s_) format\n", - " self.memory.add(sample) \n", + " self.memory.add(sample)\n", "\n", " # slowly decrease Epsilon based on our eperience\n", " self.steps += 1\n", " self.epsilon = MIN_EPSILON + (MAX_EPSILON - MIN_EPSILON) * math.exp(-LAMBDA * self.steps)\n", "\n", - " def replay(self): \n", + " def replay(self):\n", " batch = self.memory.sample(BATCH_SIZE)\n", " batchLen = len(batch)\n", "\n", " no_state = numpy.zeros(STATE_COUNT)\n", "\n", - " \n", + "\n", " # CNTK: explicitly setting to float32\n", " states = numpy.array([ o[0] for o in batch ], dtype=np.float32)\n", " states_ = numpy.array([(no_state if o[3] is None else o[3]) for o in batch ], dtype=np.float32)\n", @@ -398,10 +398,10 @@ " # CNTK: explicitly setting to float32\n", " x = numpy.zeros((batchLen, STATE_COUNT)).astype(np.float32)\n", " y = numpy.zeros((batchLen, ACTION_COUNT)).astype(np.float32)\n", - " \n", + "\n", " for i in range(batchLen):\n", " s, a, r, s_ = batch[i]\n", - " \n", + "\n", " # CNTK: [0] because of sequence dimension\n", " t = p[0][i]\n", " if s_ is None:\n", @@ -441,7 +441,7 @@ "\n", " for i, data in enumerate(weights):\n", " axi = ax if len(weights)==1 else ax[i]\n", - " if isinstance(data, tuple): \n", + " if isinstance(data, tuple):\n", " w, title = data\n", " axi.set_title(title)\n", " else:\n", @@ -492,9 +492,9 @@ "\n", "def run(agent):\n", " s = env.reset()\n", - " R = 0 \n", + " R = 0\n", "\n", - " while True: \n", + " while True:\n", " #env.render()\n", "\n", " # CNTK: explicitly setting to float32\n", @@ -506,7 +506,7 @@ " s_ = None\n", "\n", " agent.observe((s, a, r, s_))\n", - " agent.replay() \n", + " agent.replay()\n", "\n", " s = s_\n", " R += r\n", @@ -522,13 +522,13 @@ " reward_sum += run(agent)\n", " episode_number += 1\n", " if episode_number % BATCH_SIZE_BASELINE == 0:\n", - " print('Episode: %d, Average reward for episode %f.' % (episode_number, \n", + " print('Episode: %d, Average reward for episode %f.' % (episode_number,\n", " reward_sum / BATCH_SIZE_BASELINE))\n", " if episode_number%200==0:\n", " plot_weights([(agent.brain.params['W1'], 'Episode %i $W_1$'%episode_number)], figsize=(14,5))\n", " if reward_sum / BATCH_SIZE_BASELINE > REWARD_TARGET:\n", " print('Task solved in %d episodes' % episode_number)\n", - " plot_weights([(agent.brain.params['W1'], 'Episode %i $W_1$'%episode_number)], figsize=(14,5)) \n", + " plot_weights([(agent.brain.params['W1'], 'Episode %i $W_1$'%episode_number)], figsize=(14,5))\n", " break\n", " reward_sum = 0\n", "agent.brain.model.save_model('dqn.mod')" @@ -600,7 +600,7 @@ "metadata": {}, "source": [ "# Part 2: Policy gradient\n", - "**Goal:** \n", + "**Goal:**\n", "\\begin{equation}\\text{maximize } E [R | \\pi_\\theta]\n", "\\end{equation}\n", "\n", @@ -618,7 +618,7 @@ "metadata": {}, "source": [ "#### Rewards:\n", - "Remember, we get +1 reward for every time step, in which we still were in the game. \n", + "Remember, we get +1 reward for every time step, in which we still were in the game.\n", "\n", "The problem: we normally do not know, which action led to a continuation of the game, and which was actually a bad one. Our simple heuristic: actions in the beginning of the episode are good, and those towards the end are likely bad (they led to losing the game after all)." ] @@ -713,7 +713,7 @@ }, "outputs": [], "source": [ - "import cntk as C \n", + "import cntk as C\n", "\n", "TOTAL_EPISODES = 1500 if isFast else 10000\n", "\n", @@ -754,8 +754,8 @@ "\n", "loss = -C.reduce_mean(C.log(C.square(input_y - probability) + 1e-4) * advantages, axis=0, name='loss')\n", "\n", - "lr = 0.001 \n", - "lr_schedule = learning_rate_schedule(lr, UnitType.sample) \n", + "lr = 0.001\n", + "lr_schedule = learning_rate_schedule(lr, UnitType.sample)\n", "sgd = C.sgd([W1, W2], lr_schedule)\n", "\n", "gradBuffer = dict((var.name, np.zeros(shape=var.shape)) for var in loss.parameters if var.name in ['W1', 'W2', 'b1', 'b2'])\n", @@ -802,9 +802,9 @@ "\n", " # Forward pass\n", " arguments = {observations: epx, input_y: epl, advantages: discounted_epr}\n", - " state, outputs_map = loss.forward(arguments, outputs=loss.outputs, \n", + " state, outputs_map = loss.forward(arguments, outputs=loss.outputs,\n", " keep_for_backward=loss.outputs)\n", - " \n", + "\n", " # Backward psas\n", " root_gradients = {v: np.ones_like(o) for v, o in outputs_map.items()}\n", " vargrads_map = loss.backward(state, root_gradients, variables=set([W1, W2]))\n", @@ -814,12 +814,12 @@ "\n", " # Wait for some batches to finish to reduce noise\n", " if episode_number % BATCH_SIZE_BASELINE == 0:\n", - " grads = {W1: gradBuffer['W1'].astype(np.float32), \n", + " grads = {W1: gradBuffer['W1'].astype(np.float32),\n", " W2: gradBuffer['W2'].astype(np.float32)}\n", " updated = sgd.update(grads, BATCH_SIZE_BASELINE)\n", "\n", " # reset the gradBuffer\n", - " gradBuffer = dict((var.name, np.zeros(shape=var.shape)) \n", + " gradBuffer = dict((var.name, np.zeros(shape=var.shape))\n", " for var in loss.parameters if var.name in ['W1', 'W2', 'b1', 'b2'])\n", "\n", " print('Episode: %d. Average reward for episode %f.' % (episode_number, reward_sum / BATCH_SIZE_BASELINE))\n", From b3e7bb2e8b5048c39629b80170c33e857a6cff1d Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 12 Jan 2017 15:11:19 +0100 Subject: [PATCH 114/120] Python: include all wheels in drop --- Scripts/install/linux/install-cntk.sh | 6 +++--- Tests/Install/linux/test.sh | 6 +++--- Tools/make_binary_drop_linux | 2 -- Tools/make_binary_drop_windows.ps1 | 8 -------- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/Scripts/install/linux/install-cntk.sh b/Scripts/install/linux/install-cntk.sh index fb4c35dc7552..6773878289df 100755 --- a/Scripts/install/linux/install-cntk.sh +++ b/Scripts/install/linux/install-cntk.sh @@ -12,11 +12,11 @@ while [ $# -gt 0 ]; do case "$1" in --py-version) case "$2" in - 34 | 35) + 27 | 34 | 35) PY_VERSION="$2" ;; *) - echo Invalid or missing value for --py-version option, please specify 34 or 35. + echo Invalid or missing value for --py-version option, please specify 27, 34, or 35. exit 1 ;; esac @@ -46,7 +46,7 @@ CNTK_EXAMPLES_PATH="$PWD/Examples" CNTK_TUTORIALS_PATH="$PWD/Tutorials" CNTK_BINARY="$CNTK_BIN_PATH/cntk" CNTK_PY_ENV_FILE="$SCRIPT_DIR/conda-linux-cntk-py$PY_VERSION-environment.yml" -CNTK_WHEEL_PATH="cntk/python/cntk-2.0.beta7.0-cp$PY_VERSION-cp${PY_VERSION}m-linux_x86_64.whl" +CNTK_WHEEL_PATH="cntk/python/cntk-2.0.beta7.0-cp$PY_VERSION-cp${PY_VERSION}m*-linux_x86_64.whl" test -d "$CNTK_BIN_PATH" && test -d "$CNTK_LIB_PATH" && test -d "$CNTK_DEP_LIB_PATH" && test -d "$CNTK_TUTORIALS_PATH" && test -d "$CNTK_EXAMPLES_PATH" && test -x "$CNTK_BINARY" && diff --git a/Tests/Install/linux/test.sh b/Tests/Install/linux/test.sh index 62bf770415e7..166dc8783d45 100755 --- a/Tests/Install/linux/test.sh +++ b/Tests/Install/linux/test.sh @@ -1,7 +1,7 @@ #!/bin/bash set -x -e -o pipefail -USAGE="Usage: $0 [--py-version [34|35]] -- " +USAGE="Usage: $0 [--py-version [27|34|35]] -- " SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" @@ -11,11 +11,11 @@ while [ $# -gt 0 ]; do case "$1" in --py-version) case "$2" in - 34 | 35) + 27 | 34 | 35) PY_VERSION="$2" ;; *) - echo Invalid or missing value for --py-version option, please specify 34 or 35. + echo Invalid or missing value for --py-version option, please specify 27, 34, or 35. exit 1 ;; esac diff --git a/Tools/make_binary_drop_linux b/Tools/make_binary_drop_linux index 56ce9b5d06e6..63d254029dd5 100755 --- a/Tools/make_binary_drop_linux +++ b/Tools/make_binary_drop_linux @@ -173,8 +173,6 @@ cp -r $buildPath/* $baseBinariesPath # General TODO: Implement White List of Binary Drop contents. # For the time being "cherry pick" removal of unneeded files # -# Remove Py 2.7 for the time being -rm -f $baseBinariesPath/python/cntk*27*27*.whl rm -f $baseBinariesPath/bin/brainscripttests rm -f $baseBinariesPath/bin/cppevalclient rm -f $baseBinariesPath/bin/cppevalextendedclient diff --git a/Tools/make_binary_drop_windows.ps1 b/Tools/make_binary_drop_windows.ps1 index d4d29c811062..651ff6bd4193 100644 --- a/Tools/make_binary_drop_windows.ps1 +++ b/Tools/make_binary_drop_windows.ps1 @@ -80,14 +80,6 @@ Remove-Item $baseDropPath\cntk\*.lib -Exclude EvalDll.lib, CNTKLibrary-2.0.lib Remove-Item $baseDropPath\cntk\*.exp Remove-Item $baseDropPath\cntk\*.metagen # Remove specific items -If (Test-Path $baseDropPath\cntk\Python\cntk-*-cp27*.whl) -{ - Remove-Item $baseDropPath\cntk\Python\cntk-*-cp27*.whl -} -If (Test-Path $baseDropPath\cntk\Python\cntk-*-cp35*.whl) -{ - Remove-Item $baseDropPath\cntk\Python\cntk-*-cp35*.whl -} If (Test-Path $baseDropPath\cntk\CPPEvalClientTest.exe) { Remove-Item $baseDropPath\cntk\CPPEvalClientTest.exe From 88b6c328a82edb33963a5487b51af3873f1320be Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 12 Jan 2017 15:56:52 +0100 Subject: [PATCH 115/120] Address CR comments --- .../Examples/CNTK_203_Reinforcement_Learning_Basics_test.py | 6 +++++- Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py index cb62638c76c8..47fdbfbbfb61 100644 --- a/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py +++ b/Tests/EndToEndTests/CNTKv2Python/Examples/CNTK_203_Reinforcement_Learning_Basics_test.py @@ -17,5 +17,9 @@ def test_cntk_203_reinforcement_learning_basics_noErrors(nb): assert errors == [] def test_cntk_203_reinforcement_learning_basics_tasks_are_solved(nb): - testCells = [cell for cell in nb.cells if cell.cell_type == 'code' and len(cell.outputs) > 0 and 'text' in cell.outputs[0] and re.search('Task solved in[ :]', cell.outputs[0]['text'])] + testCells = [cell for cell in nb.cells + if cell.cell_type == 'code' and + len(cell.outputs) > 0 and + 'text' in cell.outputs[0] and + re.search('Task solved in[ :]', cell.outputs[0]['text'])] assert len(testCells) == 2 diff --git a/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb b/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb index 0e5de472d8ac..6b5f19b497c7 100644 --- a/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb +++ b/Tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb @@ -198,7 +198,8 @@ "#from keras.optimizers import *\n", "from cntk import *\n", "from cntk.models import Sequential\n", - "from cntk.layers import *# Select the right target device when this notebook is being tested:\n", + "from cntk.layers import *\n", + "# Select the right target device when this notebook is being tested:\n", "if 'TEST_DEVICE' in os.environ:\n", " if os.environ['TEST_DEVICE'] == 'cpu':\n", " device.set_default_device(device.cpu())\n", From 38f68fba99afc61c84e153a4d84f873a8a1d7d19 Mon Sep 17 00:00:00 2001 From: Chris Basoglu Date: Thu, 12 Jan 2017 08:54:05 -0800 Subject: [PATCH 116/120] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index b0b7bfa33cd8..b4f693d5a98d 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ **The [CNTK Wiki](https://github.com/Microsoft/CNTK/wiki) has all information on CNTK including [setup](https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-your-machine), [examples](https://github.com/Microsoft/CNTK/wiki/Examples), etc.** +Give us feedback through these [channels](https://github.com/Microsoft/CNTK/wiki/Feedback-Channels) + # Latest news ***2017-01-10.*** CNTK for Windows supports Visual 2015 From 3584f63a693d60991e5fc1d7dd4b58c9558b2f2f Mon Sep 17 00:00:00 2001 From: Chris Basoglu Date: Thu, 12 Jan 2017 08:54:31 -0800 Subject: [PATCH 117/120] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b4f693d5a98d..3d0174f9f424 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ **The [CNTK Wiki](https://github.com/Microsoft/CNTK/wiki) has all information on CNTK including [setup](https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-your-machine), [examples](https://github.com/Microsoft/CNTK/wiki/Examples), etc.** -Give us feedback through these [channels](https://github.com/Microsoft/CNTK/wiki/Feedback-Channels) +Give us feedback through these [channels](https://github.com/Microsoft/CNTK/wiki/Feedback-Channels). # Latest news From 2a75e21aac03e295b074ff68dc6ef156c91d40c0 Mon Sep 17 00:00:00 2001 From: Mark Hillebrand Date: Thu, 12 Jan 2017 19:40:34 +0000 Subject: [PATCH 118/120] Scripts/install/linux/install-cntk.sh: fix wheel name --- Scripts/install/linux/install-cntk.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Scripts/install/linux/install-cntk.sh b/Scripts/install/linux/install-cntk.sh index 6773878289df..f428ff4d2a41 100755 --- a/Scripts/install/linux/install-cntk.sh +++ b/Scripts/install/linux/install-cntk.sh @@ -39,6 +39,9 @@ SCRIPT_DIR="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")")" # Go to the drop root cd "$SCRIPT_DIR/../../.." +PYWHEEL_QUALIFIER=cp$PY_VERSION-cp${PY_VERSION}m +[ $PY_VERSION = 27 ] && PYWHEEL_QUALIFIER+=u + CNTK_BIN_PATH="$PWD/cntk/bin" CNTK_LIB_PATH="$PWD/cntk/lib" CNTK_DEP_LIB_PATH="$PWD/cntk/dependencies/lib" @@ -46,7 +49,8 @@ CNTK_EXAMPLES_PATH="$PWD/Examples" CNTK_TUTORIALS_PATH="$PWD/Tutorials" CNTK_BINARY="$CNTK_BIN_PATH/cntk" CNTK_PY_ENV_FILE="$SCRIPT_DIR/conda-linux-cntk-py$PY_VERSION-environment.yml" -CNTK_WHEEL_PATH="cntk/python/cntk-2.0.beta7.0-cp$PY_VERSION-cp${PY_VERSION}m*-linux_x86_64.whl" +CNTK_WHEEL_PATH="cntk/python/cntk-2.0.beta7.0-$PYWHEEL_QUALIFIER-linux_x86_64.whl" + test -d "$CNTK_BIN_PATH" && test -d "$CNTK_LIB_PATH" && test -d "$CNTK_DEP_LIB_PATH" && test -d "$CNTK_TUTORIALS_PATH" && test -d "$CNTK_EXAMPLES_PATH" && test -x "$CNTK_BINARY" && From 16a6dd305c0c82ea39bcff08951a4254ed977317 Mon Sep 17 00:00:00 2001 From: Cha Zhang Date: Thu, 12 Jan 2017 11:53:45 -0800 Subject: [PATCH 119/120] Bug in dropout rate. --- .../Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk b/Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk index 6fdbc58d4af7..594b8854714a 100644 --- a/Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk +++ b/Examples/Image/Classification/GoogLeNet/BrainScript/InceptionV3.cntk @@ -64,7 +64,7 @@ Train = { epochSize = 256000 maxEpochs = 1 minibatchSize = 128 # 16 GPU - dropoutRate = 0.8 + dropoutRate = 0.2 learningRatesPerMB = 1 momentumAsTimeConstant = 4096 From 1a176933b3dd3b96121c59c4d6baae526427d99b Mon Sep 17 00:00:00 2001 From: Amit Agarwal Date: Thu, 12 Jan 2017 15:42:52 -0800 Subject: [PATCH 120/120] CNTK v2 library: Fixed a memory leak in model serialization --- Source/CNTKv2LibraryDll/API/CNTKLibrary.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h index 186dd83023e1..ea836f15bdf6 100644 --- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h +++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h @@ -1338,7 +1338,7 @@ namespace CNTK FreePtrAsType>(); else if (m_valueType == Type::Dictionary) FreePtrAsType(); - else if (m_valueType == Type::Dictionary) + else if (m_valueType == Type::NDArrayView) FreePtrAsType(); }