Skeleton of model

Author: reyoung

paddle/framework/pybind.cc

@@ -225,7 +225,14 @@ All parameter, weight, gradient are variables in Paddle.
   py::class_<operators::NetOp, OperatorBase>(m, "Net")
       .def(py::init<>())
       .def("add_op", [](operators::NetOp &self,
-                        const OperatorBase &op) { self.AddOp(op); })
+                        const OperatorBase &op) { return self.AddOp(op); })
+      .def("__len__", &operators::NetOp::Size)
+      .def("infer_shape",
+           [](operators::NetOp &self, size_t begin, const Scope &scope) {
+             self.InferShape(begin, std::numeric_limits<size_t>::max(), scope);
+           })
+      .def("infer_shape", [](operators::NetOp &self,
+                             const Scope &scope) { self.InferShape(scope); })
       .def("complete_add_op", &operators::NetOp::CompleteAddOp)
       .def("complete_add_op",
            [](operators::NetOp &self) { self.CompleteAddOp(); })

paddle/operators/net_op.h

@@ -56,11 +56,18 @@ class NetOp : public framework::OperatorBase {
    * before every mini-batch
    */
   void InferShape(const framework::Scope& scope) const override {
-    for (auto& op : ops_) {
-      op->InferShape(scope);
+    InferShape(0, std::numeric_limits<size_t>::max(), scope);
+  }
+
+  void InferShape(size_t begin, size_t end,
+                  const framework::Scope& scope) const {
+    for (; begin < std::min(end, ops_.size()); ++begin) {
+      ops_[begin]->InferShape(scope);
     }
   }
 
+  size_t Size() const { return ops_.size(); }
+
   /**
    * @brief Run the network.
    *
@@ -84,15 +91,16 @@ class NetOp : public framework::OperatorBase {
     return true;
   }
 
-  void AddOp(const framework::OperatorBase& op) { AddOp(op.Clone()); }
+  size_t AddOp(const framework::OperatorBase& op) { return AddOp(op.Clone()); }
 
   /**
    * @brief Add an operator by ptr
    */
-  void AddOp(std::unique_ptr<framework::OperatorBase> op) {
+  size_t AddOp(std::unique_ptr<framework::OperatorBase> op) {
     PADDLE_ENFORCE(!add_op_done_, "Cannot AddOp when this network is sealed");
     PADDLE_ENFORCE_NOT_NULL(op, "Cannot Insert Null op");
     ops_.push_back(std::move(op));
+    return ops_.size() - 1;
   }
 
   void InsertOp(size_t pos, std::unique_ptr<framework::OperatorBase> op) {

python/paddle/v2/framework/layers.py

@@ -0,0 +1,71 @@
+from paddle.v2.framework.model import *
+import numpy
+
+
+def is_list_like(o):
+    return isinstance(o, list) or isinstance(o, tuple)
+
+
+def data(name, shape, model=None):
+    if model is None:
+        model = g_model
+
+    if isinstance(shape, int):
+        shape = [shape]
+
+    if not is_list_like(shape):
+        raise ValueError()
+
+    if id(model.cur_scope) != id(model.global_scope):
+        raise ValueError("Data Layer must be declared in global scope")
+
+    tensor = model.cur_scope.new_var(name).get_tensor()
+    tensor.set_dims([1] + shape)
+    return name
+
+
+def fc(input,
+       size,
+       name=None,
+       param_attr=None,
+       bias_attr=False,
+       act="sigmoid",
+       model=None):
+    if model is None:
+        model = g_model
+
+    if name is None:
+        name = model.next_name('fc')
+
+    if param_attr is None:
+        param_attr = ParameterAttribute.default_weight_attr()
+
+    dim = model.cur_scope.find_var(input).get_tensor().get_dims()
+    w = model.create_parameter(name + ".weight.", param_attr, [dim[1], size])
+
+    tmp = model.add_op_and_infer_shape("mul", X=input, Y=w)
+
+    if bias_attr is None or bias_attr is True:
+        bias_attr = ParameterAttribute.default_bias_attr()
+    if bias_attr:
+        b = model.create_parameter(name + ".bias.", bias_attr, [size])
+        tmp = model.add_op_and_infer_shape('rowwise_add', X=tmp, b=b)
+
+    if act:
+        tmp = model.add_op_and_infer_shape(act, X=tmp)
+
+    return tmp
+
+
+if __name__ == '__main__':
+    x = data("X", shape=784)
+    hidden = fc(x, size=100, bias_attr=True)
+    hidden = fc(hidden, size=100, bias_attr=True)
+    prediction = fc(hidden, size=10, bias_attr=True, act='softmax')
+
+    g_model.init_parameters()
+    g_model.feed_data({"X": numpy.random.random((1000, 784)).astype('float32')})
+
+    for i in xrange(1000):
+        g_model.run()
+        print numpy.array(g_model.find_tensor(prediction)).mean()

python/paddle/v2/framework/model.py

@@ -0,0 +1,147 @@
+import paddle.v2.framework.network as network
+import paddle.v2.framework.core as core
+
+__all__ = ['Model', 'g_model', 'ParameterAttribute']
+
+
+class ParameterAttribute(object):
+    def __init__(self,
+                 name=None,
+                 initial_max=None,
+                 initial_min=None,
+                 initial_mean=None,
+                 initial_std=None,
+                 initial_seed=0):
+        self.name = name
+
+        if initial_min is None and initial_max is None and initial_mean is None and initial_std is None:
+            initial_min = -1.0
+            initial_max = 1.0
+
+        if initial_max is not None and initial_min is not None:
+            self.init_strategy = ('uniform_random', {
+                'min': initial_min,
+                'max': initial_max,
+                'seed': initial_seed
+            })
+        elif initial_mean is not None and initial_std is not None:
+            self.init_strategy = ('gauss_random', {
+                'mean': initial_mean,
+                'std': initial_std,
+                'seed': initial_seed
+            })
+        else:
+            raise ValueError()
+
+    @staticmethod
+    def default_weight_attr():
+        return ParameterAttribute()
+
+    @staticmethod
+    def default_bias_attr():
+        # TODO(yy): Change it to FillZero.
+        return ParameterAttribute(initial_min=-0.0001, initial_max=0.0001)
+
+
+class Model(object):
+    def __init__(self, place=None):
+        self.init_network = network.Network()
+        self.network = network.Network()
+
+        if place is None:
+            self.device_context = None
+            self.place = None
+        else:
+            self.device_context = core.DeviceContext.create(place)
+            self.place = None
+
+        self.global_scope = core.Scope()
+        self.cur_scope = self.global_scope
+        self.name_counter = 0
+        self.all_param_names = set()
+        self.has_been_run = False
+
+    def next_name(self, prefix):
+        name = prefix + str(self.name_counter)
+        self.name_counter += 1
+        return name
+
+    def create_parameter(self, name_prefix, param_attr, dims):
+        if not isinstance(param_attr, ParameterAttribute):
+            raise TypeError()
+        if param_attr.name is None:
+            param_attr.name = self.next_name(name_prefix)
+
+        if self.cur_scope.find_var(param_attr.name) is not None:
+            raise ValueError("Parameter {} has been created before",
+                             param_attr.name)
+
+        self.cur_scope.new_var(param_attr.name).get_tensor()
+
+        op_type, attrs = param_attr.init_strategy
+        attrs['dims'] = dims
+        attrs['Out'] = param_attr.name
+
+        op_func = getattr(self.init_network, op_type)
+        pname = op_func(**attrs)
+        self.all_param_names.add(pname)
+
+        self.init_network.infer_shape(
+            len(self.init_network) - 1, self.cur_scope)
+
+        return pname
+
+    def add_op_and_infer_shape(self, op_type, **kwargs):
+        out = self.network.create_and_add_op(op_type, **kwargs)
+        return_value = out
+        if out is None:
+            return
+
+        if isinstance(out, unicode) or isinstance(out, str):
+            out = [out]
+
+        for o in out:
+            v = self.cur_scope.find_var(o)
+            if v is None:
+                v = self.cur_scope.new_var(o)
+            v.get_tensor()
+
+        op_idx = len(self.network) - 1
+        self.network.infer_shape(op_idx, self.cur_scope)
+
+        return return_value
+
+    def set_place(self, place):
+        if self.has_been_run:
+            raise ValueError("Cannot set place to model after run")
+
+        self.device_context = core.DeviceContext.create(place)
+        self.place = place
+
+    def init_parameters(self):
+        if self.device_context is None:
+            # TODO(yy): Log warning here
+            self.set_place(core.CPUPlace())
+        if not self.has_been_run:
+            self.has_been_run = True
+
+        self.init_network.run(self.global_scope, self.device_context)
+
+    def run(self):
+        if not self.has_been_run:
+            self.has_been_run = True
+        self.network.infer_shape(self.global_scope)
+        self.network.run(self.global_scope, self.device_context)
+
+    def feed_data(self, data):
+        for key in data:
+            tensor = self.global_scope.find_var(key).get_tensor()
+            d = data[key]
+            tensor.set_dims(d.shape)
+            tensor.set(d, self.place)
+
+    def find_tensor(self, var_name):
+        return self.global_scope.find_var(var_name).get_tensor()
+
+
+g_model = Model()

python/paddle/v2/framework/network.py

@@ -30,6 +30,9 @@ def __impl__(*args, **kwargs):
             else:
                 raise AttributeError("No such attribute %s" % name)
 
+    def create_and_add_op(self, type, **kwargs):
+        return getattr(self, type)(**kwargs)
+
     def add_op(self, op):
         if isinstance(op, Network):
             self.add_op(op.net)
@@ -38,3 +41,6 @@ def add_op(self, op):
 
     def __str__(self):
         return str(self.net)
+
+    def __len__(self):
+        return len(self.net)

python/paddle/v2/framework/op.py

@@ -9,9 +9,9 @@ def get_all_op_protos():
     Get all registered op proto from Paddle C++
     :return: list of OpProto
     """
+    global g_all_op_protos
     protostrs = core.get_all_op_protos()
     if g_all_op_protos is None:
-        global g_all_op_protos
         g_all_op_protos = dict()
         for pbstr in protostrs:
             op_proto = framework_pb2.OpProto.FromString(str(pbstr))