Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Make more C++ unit tests work for batch norm #28

Merged
merged 33 commits into from
Feb 12, 2018
Merged

Make more C++ unit tests work for batch norm #28

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
e4191b6
WIP unit tests
Feb 6, 2018
af393cc
some backward items initialized
Feb 7, 2018
a07b2fd
some backward items initialized
Feb 7, 2018
fdfa744
some backward items initialized
Feb 7, 2018
3ef852e
first unit test working
Feb 7, 2018
fc23fc1
Working on types
Feb 7, 2018
452ecce
backward types working for fp16 on first unit test
Feb 8, 2018
6ef95f5
backward types working for fp16 on first unit test
Feb 8, 2018
8a97b32
backward types working for fp16 on first unit test
Feb 8, 2018
0505423
.
Feb 8, 2018
1e73b6c
.
Feb 8, 2018
385037f
some tests working
Feb 8, 2018
e103555
fix input data
Feb 8, 2018
6d7be19
hangle gpu<->cpu for setting values
Feb 9, 2018
508783d
gpu working
Feb 9, 2018
f981878
gpu working
Feb 9, 2018
7938b45
CAccessAsCPU class
Feb 9, 2018
d4720bc
Fix varying type in AccessAsCPU
Feb 9, 2018
d1d8d45
starting to add channel axis tests
Feb 9, 2018
ffb5b93
TestChannelAxisSimple
Feb 9, 2018
1546add
TestChannelAxisSimple
Feb 9, 2018
d1fa2a1
run bidirectional
Feb 9, 2018
fa27dbd
run bidirectional
Feb 9, 2018
3c3d7a8
run bidirectional
Feb 9, 2018
2a3be06
CLEANUP
Feb 9, 2018
891a56f
CLEANUP
Feb 9, 2018
7f5f71a
..
Feb 9, 2018
4e61178
noaxis
Feb 9, 2018
8a2bdbc
..
Feb 10, 2018
17c6a00
lint
Feb 12, 2018
27b5b04
revert
Feb 12, 2018
c3a54d7
revert
Feb 12, 2018
5a29075
Merge remote-tracking branch 'zd/mkldnn' into myzdmkl
Feb 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
run bidirectional
  • Loading branch information
Olivier committed Feb 9, 2018
commit 3c3d7a8f872c651260b98748fd44da4d1b735ee6
18 changes: 9 additions & 9 deletions src/operator/nn/batch_norm.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -189,8 +189,8 @@ void BatchNormForwardImpl(mshadow::Stream<cpu> *,
}
}
}
PRT("FWD runningMean", runningMean);
PRT("FWD runningVariance", runningVariance);
//PRT("FWD runningMean", runningMean);
//PRT("FWD runningVariance", runningVariance);
}

template <typename xpu, typename DType, typename AccReal>
Expand All @@ -206,30 +206,30 @@ void BatchNormBackwardImpl(mshadow::Stream<cpu> *,
batchnorm::BNTensor3<DType> inputData(in_data[batchnorm::kData], param_.axis);
const TBlob &weights = in_data[batchnorm::kGamma];

PRT("weights", weights);
//PRT("weights", weights);

// Input Grad
batchnorm::BNTensor3<DType> gradIn(in_grad[batchnorm::kData], param_.axis);
const TBlob &gradWeight = in_grad[batchnorm::kGamma];
const TBlob &gradBias = in_grad[batchnorm::kBeta];

PRT("gradWeight", gradWeight);
PRT("gradBias", gradBias);
//PRT("gradWeight", gradWeight);
//PRT("gradBias", gradBias);

// Aux (Moving)
const TBlob &runningMean = aux_states[batchnorm::kMovingMean];
const TBlob &runningVariance = aux_states[batchnorm::kMovingVar];

PRT("runningMean", runningMean);
PRT("runningVariance", runningVariance);
//PRT("runningMean", runningMean);
//PRT("runningVariance", runningVariance);

// Output
batchnorm::BNTensor3<DType> gradOut(out_grad[batchnorm::kOut], param_.axis);
const TBlob &saveMean = out_data[batchnorm::kMean];
const TBlob &saveStd = out_data[batchnorm::kVar];

PRT("saveMean", saveMean);
PRT("saveStd", saveStd);
//PRT("saveMean", saveMean);
//PRT("saveStd", saveStd);

const size_t channelCount = inputData.ChannelCount();
const size_t itemCount = inputData.Size() / channelCount;
Expand Down
36 changes: 2 additions & 34 deletions tests/cpp/include/test_core_op.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,12 +276,6 @@ class CoreOpExecutor : public test::op::OperatorDataInitializer<DType>
std::map<int, const NDArray *>& index2array) const {
index2array.clear();
static auto& fgradient = nnvm::Op::GetAttr<nnvm::FGradient>("FGradient");
// std::vector<bool>& save_inputs = *p_save_inputs;
// std::vector<bool>& save_outputs = *p_save_outputs;
// save_inputs.resize(num_inputs);
// save_outputs.resize(num_outputs);
// std::fill(save_inputs.begin(), save_inputs.end(), false);
// std::fill(save_outputs.begin(), save_outputs.end(), false);

const uint32_t num_inputs = inputs().size();
const uint32_t num_outputs = outputs().size();
Expand All @@ -306,24 +300,6 @@ class CoreOpExecutor : public test::op::OperatorDataInitializer<DType>
if (!igrad_entries.empty()) {
return igrad_entries[0].node;
}

// for (const auto& i : igrad_entries) {
// if (i.node == nullptr && i.version == 0) {
// save_inputs[i.index] = true;
// } else if (i.node == node) {
// save_outputs[i.index] = true;
// }
// }
// DFSVisit(igrad_entries, [&](const nnvm::NodePtr& gnode) {
// if (!gnode || gnode == node) return;
// for (const auto& i : gnode->inputs) {
// if (i.node == nullptr && i.version == 0) {
// save_inputs[i.index] = true;
// } else if (i.node == node) {
// save_outputs[i.index] = true;
// }
// }
// });
}
return nullptr;
}
Expand Down Expand Up @@ -550,16 +526,6 @@ class CoreOpExecutor : public test::op::OperatorDataInitializer<DType>
if (!backward_for_op) {
DispatchMode dispatch_mode = DispatchMode::kUndefined;
imperative::SetShapeType(ctx_.run_ctx.ctx, attrs_, inputs_p, outputs_p, &dispatch_mode);
} else {
// Backward op, so set based upon inputs
//CHECK_EQ(static_cast<size_t>(num_visible_outputs), backward_for_op->inputs().size());
// for (int i = 0; i < num_visible_outputs; ++i) {
// CHECK_LT(static_cast<size_t>(i), input_shapes.size());
// // backward outputs should look like forward inputs
// // TODO(cjolivier01): This check fails for dot product...
// // Need better inference of backward shapes
// // CHECK_EQ(backward_for_op->inputs()[i].shape(), outputs_[i].shape());
// }
}

std::vector<OpReqType> req;
Expand Down Expand Up @@ -640,6 +606,8 @@ class CoreOpExecutor : public test::op::OperatorDataInitializer<DType>
void Execute() {
CHECK_EQ(initialized_, true);
CHECK_NOTNULL(function_);
CollectBlobs(inputs_, &blob_inputs_);
CollectBlobs(outputs_, &blob_outputs_);
function_(attrs_, ctx_, blob_inputs_, req_, blob_outputs_);
}

Expand Down
43 changes: 28 additions & 15 deletions tests/cpp/operator/batchnorm_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ enum BackwardInputs {
};
enum BackwardOutputs {
/* in_grad */ bwd_in_grad_Data /* Original input data */,
bwd_in_grad_Gamma, bwd_in_grad_Beta
/* weight, bias*/ bwd_in_grad_Gamma, bwd_in_grad_Beta
};

/**
Expand Down Expand Up @@ -214,9 +214,9 @@ class BNOperatorExecutor : public test::op::CoreOpExecutor<DType, AccReal> {
Super::resetBackward();

// Join forward input and in_data array
test::print(ctx().run_ctx, &std::cout, GetBlob(bwd_in_data_Data));
//test::print(ctx().run_ctx, &std::cout, GetBlob(bwd_in_data_Data));
*GetArray(bwd_in_data_Data) = *GetArray(kForInData);
test::print(ctx().run_ctx, &std::cout, GetBlob(bwd_in_data_Data));
//test::print(ctx().run_ctx, &std::cout, GetBlob(bwd_in_data_Data));

*GetArray(bwd_in_data_Gamma) = *GetArray(kForGamma);
*GetArray(bwd_in_data_Beta) = *GetArray(kForBeta);
Expand All @@ -234,9 +234,8 @@ class BNOperatorExecutor : public test::op::CoreOpExecutor<DType, AccReal> {
*GetArray(bwd_aux_states_MovingMean) = *GetArray(kForMovingMean);
*GetArray(bwd_aux_states_MovingVar) = *GetArray(kForMovingVar);

test::print(ctx().run_ctx, &std::cout, GetBlob(bwd_aux_states_MovingMean));
test::print(ctx().run_ctx, &std::cout, GetBlob(bwd_aux_states_MovingVar));

//test::print(ctx().run_ctx, &std::cout, GetBlob(bwd_aux_states_MovingMean));
//test::print(ctx().run_ctx, &std::cout, GetBlob(bwd_aux_states_MovingVar));

double val = -.101;
test::patternFill(ctx().run_ctx, &GetBlob(bwd_out_data_Data), [&val]() -> double {
Expand Down Expand Up @@ -618,15 +617,14 @@ static StreamType& dumpB(StreamType *os,
*os << "=============================" << std::endl;
}

// typedef typename OperatorExecutor::BlobVectorType BlobVectorType;
// DBPRT(os, *prop.executor_, BlobVectorType::kInGrad, mxnet::op::batchnorm::kData);
// DBPRT(os, *prop.executor_, BlobVectorType::kInGrad, mxnet::op::batchnorm::kGamma);
// DBPRT(os, *prop.executor_, BlobVectorType::kInGrad, mxnet::op::batchnorm::kBeta);
//
// DBPRT(os, *prop.executor_, BlobVectorType::kAux, mxnet::op::batchnorm::kMovingMean);
// DBPRT(os, *prop.executor_, BlobVectorType::kAux, mxnet::op::batchnorm::kMovingVar);
//
// DBPRT(os, *prop.executor_, BlobVectorType::bwd_out_grad_Grad, mxnet::op::batchnorm::kOut);
DBPRT(os, *prop.executor_, BackwardOutputs::bwd_in_grad_Data);
DBPRT(os, *prop.executor_, BackwardOutputs::bwd_in_grad_Gamma);
DBPRT(os, *prop.executor_, BackwardOutputs::bwd_in_grad_Beta);

DBPRT(os, *prop.executor_, BackwardInputs::bwd_aux_states_MovingMean);
DBPRT(os, *prop.executor_, BackwardInputs::bwd_aux_states_MovingVar);

DBPRT(os, *prop.executor_, BackwardInputs::bwd_out_grad_Grad);
}
return *os;
}
Expand Down Expand Up @@ -1398,11 +1396,26 @@ static void runChannelAxisTest(

// Run both operators forward and backwards several times
for (index_t x = 0; x < numberOfPasses; ++x) {
// dumpF(&std::cout, info_c1, 1);
// dumpF(&std::cout, info_c2, 2);
// dumpB(&std::cout, info_c2, 1);
// dumpB(&std::cout, info_c1, 2);

info_c1.executor_->forward(1);
info_c2.executor_->forward(1);

// dumpF(&std::cout, info_c1, 1);
// dumpF(&std::cout, info_c2, 2);
// dumpB(&std::cout, info_c2, 1);
// dumpB(&std::cout, info_c1, 2);

info_c1.executor_->backward(1);
info_c2.executor_->backward(1);

// dumpF(&std::cout, info_c1, 1);
// dumpF(&std::cout, info_c2, 2);
// dumpB(&std::cout, info_c2, 1);
// dumpB(&std::cout, info_c1, 2);
break; // REMOVE ME
}

Expand Down