add book04.word2vec train test

paddle/framework/var_desc.cc

@@ -18,6 +18,10 @@ limitations under the License. */
 namespace paddle {
 namespace framework {
 
+VarDesc::VarType VarDescBind::GetType() const { return desc_.type(); }
+
+void VarDescBind::SetType(VarDesc::VarType type) { desc_.set_type(type); }
+
 void VarDescBind::SetShape(const std::vector<int64_t> &dims) {
   VectorToRepeated(dims, mutable_tensor_desc()->mutable_dims());
 }

paddle/framework/var_desc.h

@@ -75,9 +75,9 @@ class VarDescBind {
 
   int32_t GetLodLevel() const;
 
-  VarDesc::VarType GetType() const { return desc_.type(); }
+  VarDesc::VarType GetType() const;
 
-  void SetType(VarDesc::VarType type) { desc_.set_type(type); }
+  void SetType(VarDesc::VarType type);
 
   bool Persistable() const { return desc_.persistable(); }
 

paddle/operators/lookup_table_op.cc

@@ -32,6 +32,9 @@ class LookupTableOp : public framework::OperatorWithKernel {
     auto table_dims = ctx->GetInputDim("W");
     auto ids_dims = ctx->GetInputDim("Ids");
 
+    PADDLE_ENFORCE_EQ(ids_dims.size(), 2);
+    PADDLE_ENFORCE_EQ(ids_dims[1], 1);
+
     ctx->SetOutputDim("Out", {ids_dims[0], table_dims[1]});
     ctx->ShareLoD("Ids", /*->*/ "Out");
   }
@@ -53,7 +56,9 @@ class LookupTableOpMaker : public framework::OpProtoAndCheckerMaker {
              " which is a learnable parameter.");
     AddInput("Ids",
              "An input with type int32 or int64"
-             "contains the ids to be looked up in W.");
+             "contains the ids to be looked up in W."
+             "Ids must be a column vector with rank = 2."
+             "The 2nd dimension size must be 1");
     AddOutput("Out", "The lookup results, which have the same type with W.");
     AddComment(R"DOC(
 This operator is used to perform lookups on the parameter W,

paddle/pybind/protobuf.cc

@@ -257,6 +257,7 @@ void BindOpDesc(py::module &m) {
       .def("block_attr", &OpDescBind::GetBlockAttr)
       .def("check_attrs", &OpDescBind::CheckAttrs)
       .def("infer_shape", &OpDescBind::InferShape)
+      .def("infer_var_type", &OpDescBind::InferVarType)
       .def("serialize_to_string", [](OpDescBind &op_desc) -> py::bytes {
         const OpDesc *desc = op_desc.Proto();
         PADDLE_ENFORCE(desc->IsInitialized(),

python/paddle/v2/framework/framework.py

@@ -53,8 +53,8 @@ def __init__(self,
             if is_new_var:
                 self.desc.set_data_type(dtype)
             else:
-                old_dtype = self.data_type()
-                if dtype != old_shape:
+                old_dtype = self.data_type
+                if dtype != old_dtype:
                     raise ValueError("Variable {0} has been created before. "
                                      "The previous data type is {1}; the new "
                                      "data type is {2}. They are not "
@@ -191,7 +191,6 @@ def __init__(self,
                 "`type` to initilized an Operator can not be None.")
         self.desc.set_type(type)
         proto = OpProtoHolder.instance().get_op_proto(type)
-
         if inputs is not None:
             given = set()
             need = set()
@@ -206,6 +205,7 @@ def __init__(self,
                         str(e) for e in given)))
 
             for in_proto in proto.inputs:
+
                 in_argus = inputs[in_proto.name]
                 if not isinstance(in_argus, list):
                     in_argus = [in_argus]
@@ -257,6 +257,7 @@ def __init__(self,
 
         self.desc.check_attrs()
         if type not in {'feed', 'fetch'}:
+            self.desc.infer_var_type(self.block.desc)
             self.desc.infer_shape(self.block.desc)
 
     def __str__(self):

python/paddle/v2/framework/layer_helper.py

@@ -120,10 +120,7 @@ def create_parameter(self, attr, shape, dtype, suffix='w'):
         if attr['name'] is None:
             attr['name'] = unique_name(".".join([self.name, suffix]))
         self.init_program.global_block().create_parameter(
-            name=attr['name'],
-            dtype=dtype,
-            shape=shape,
-            init_attr=attr['init_attr'])
+            dtype=dtype, shape=shape, **attr)
         return self.program.global_block().create_parameter(
             name=attr['name'], dtype=dtype, shape=shape)
 

python/paddle/v2/framework/layers.py

@@ -3,7 +3,9 @@
 from paddle.v2.framework.framework import OpProtoHolder, Variable
 import re
 
-__all__ = ['fc', 'data', 'cross_entropy', 'conv2d', 'pool2d']
+__all__ = [
+    'fc', 'data', 'cross_entropy', 'conv2d', 'pool2d', 'embedding', 'concat'
+]
 
 
 def fc(input,
@@ -55,6 +57,24 @@ def fc(input,
     return helper.append_activation(pre_activation)
 
 
+def embedding(input,
+              size,
+              data_type='float32',
+              param_attr=None,
+              program=None,
+              init_program=None):
+    helper = LayerHelper('embedding', **locals())
+    w = helper.create_parameter(
+        attr=helper.param_attr, shape=size, dtype=data_type)
+    tmp = helper.create_tmp_variable(data_type)
+    helper.append_op(
+        type='lookup_table',
+        inputs={'Ids': input,
+                'W': w},
+        outputs={'Out': tmp})
+    return tmp
+
+
 def data(name,
          shape,
          data_type='float32',
@@ -122,6 +142,19 @@ def func(**kwargs):
 _create_op_func_('mul')
 
 
+def concat(input, axis, program=None, init_program=None):
+    helper = LayerHelper('concat', **locals())
+    if not isinstance(input, list) and not isinstance(input, tuple):
+        input = [input]
+    out = helper.create_tmp_variable(dtype=input[0].data_type)
+    helper.append_op(
+        type='concat',
+        inputs={'X': input},
+        outputs={'Out': [out]},
+        attrs={'axis': axis})
+    return out
+
+
 def cross_entropy(input, label, **kwargs):
     helper = LayerHelper('cross_entropy', **kwargs)
     out = helper.create_tmp_variable(dtype=input.data_type)

python/paddle/v2/framework/tests/test_layers.py

@@ -88,6 +88,77 @@ def test_recognize_digits_conv(self):
 
         print str(program)
 
+    def test_word_embedding(self):
+        program = Program()
+        dict_size = 10000
+        embed_size = 32
+        first_word = layers.data(
+            name='firstw', shape=[1], data_type='int32', program=program)
+        second_word = layers.data(
+            name='secondw', shape=[1], data_type='int32', program=program)
+        third_word = layers.data(
+            name='thirdw', shape=[1], data_type='int32', program=program)
+        forth_word = layers.data(
+            name='forthw', shape=[1], data_type='int32', program=program)
+        next_word = layers.data(
+            name='nextw', shape=[1], data_type='int32', program=program)
+
+        embed_param_attr_1 = {
+            'name': 'shared_w',
+            'init_attr': {
+                'max': 1.0,
+                'type': 'uniform_random',
+                'min': -1.0
+            }
+        }
+        embed_param_attr_2 = {'name': 'shared_w'}
+
+        embed_first = layers.embedding(
+            input=first_word,
+            size=[dict_size, embed_size],
+            data_type='float32',
+            param_attr=embed_param_attr_1,
+            program=program)
+        embed_second = layers.embedding(
+            input=second_word,
+            size=[dict_size, embed_size],
+            data_type='float32',
+            param_attr=embed_param_attr_2,
+            program=program)
+
+        embed_third = layers.embedding(
+            input=third_word,
+            size=[dict_size, embed_size],
+            data_type='float32',
+            param_attr=embed_param_attr_2,
+            program=program)
+        embed_forth = layers.embedding(
+            input=forth_word,
+            size=[dict_size, embed_size],
+            data_type='float32',
+            param_attr=embed_param_attr_2,
+            program=program)
+
+        concat_embed = layers.concat(
+            input=[embed_first, embed_second, embed_third, embed_forth],
+            axis=1,
+            program=program)
+
+        hidden1 = layers.fc(input=concat_embed,
+                            size=256,
+                            act='sigmoid',
+                            program=program)
+        predict_word = layers.fc(input=hidden1,
+                                 size=dict_size,
+                                 act='softmax',
+                                 program=program)
+        cost = layers.cross_entropy(
+            input=predict_word, label=next_word, program=program)
+        avg_cost = layers.mean(x=cost, program=program)
+        self.assertIsNotNone(avg_cost)
+
+        print str(program)
+
 
 if __name__ == '__main__':
     unittest.main()

python/paddle/v2/framework/tests/test_lookup_table_op.py

@@ -8,7 +8,8 @@ def setUp(self):
         self.op_type = "lookup_table"
         table = np.random.random((17, 31)).astype("float32")
         ids = np.random.randint(0, 17, 4).astype("int32")
-        self.inputs = {'W': table, 'Ids': ids}
+        ids_expand = np.expand_dims(ids, axis=1)
+        self.inputs = {'W': table, 'Ids': ids_expand}
         self.outputs = {'Out': table[ids]}
 
     def test_check_output(self):