Merge branch 'master' into issue#556

Author: zhanyuanucb

docs/ontology_generation.md

@@ -72,6 +72,13 @@ Let us consider a simple ontology for documents of a pet shop.
                 {
                     "name": "pet_type",
                     "type": "str"
+                },
+                {
+                    "name": "price",
+                    "description": "Price for pet. A 2x2 matrix, whose columns are female/male and rows are juvenile/adult.",
+                    "type": "NdArray",
+                    "ndarray_dtype": "float",
+                    "ndarray_shape": [2, 2]
                 }
             ]
         },
@@ -133,7 +140,7 @@ Each entry definition will define a couple (can be empty) attributes, mimicking
 * The `description` keyword is optionally used as the comment to describe the attribute.
 * The `type` keyword is used to define the type of the attribute. Currently supported types are:
     * Primitive types - `int`, `float`, `str`, `bool`
-    * Composite types - `List`, `Dict`
+    * Composite types - `List`, `Dict`, `NdArray`
     * Entries defined in the `top` module - The attributes can be of the type base
     entries (defined in the `forte.data.ontology.top` module) and can be directly 
     referred by the class name.
@@ -146,6 +153,8 @@ Each entry definition will define a couple (can be empty) attributes, mimicking
 * `key_type` and `value_type`: If the `type` of the property is a `Dict`,
    then these two represent the types of the key and value of the dictionary,
    currently, only primitive types are supported as the `key_type`.
+* `ndarray_dtype: str` and `ndarray_shape: array`: If the `type` of the property is a `NdArray`, then 
+   these two represent the data type and the shape of the array. `NdArray` allows storing a N-dimensional (N-d) array in an entry. For instance, through the simple ontology of pet shop above, we are able to instantiate `Pet` and name it `dog`. Then, we can assign a matrix to the attribute `price` by `dog.price.data = [[2.99, 1.99], [4.99, 3.99]]`. Internally, this $2 \times 2$ matrix is stored as a NumPy array. When `ndarray_shape`/`ndarray_dtype` is specified, the shape/data type of the upcoming array will be verified whether they match. If both `ndarray_dtype` and `ndarray_shape` are provided, a placeholder will be created by `numpy.ndarray(ndarray_shape, dtype=ndarray_dtype)`.
 
 ## Major ontology types, Annotations, Links, Groups and Generics
 There are some very frequently used types in NLP: 

forte/data/base_store.py

@@ -0,0 +1,161 @@
+# Copyright 2019 The Forte Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from abc import abstractmethod
+from typing import (
+    List,
+    Type,
+    Union,
+    Iterator,
+    Tuple,
+)
+
+from forte.data.ontology.core import EntryType
+
+__all__ = ["BaseStore"]
+
+
+class BaseStore:
+    r"""The base class which will be used by :class:
+    `~forte.data.data_store.DataStore`."""
+
+    def __init__(self):
+        r"""
+        This is a base class for the efficient underlying data structure. A
+        current implementation of `BaseStore` is `DataStore`.
+
+        A `BaseStore` contains a collection of Forte entries.
+        Each entry type contains some subtypes, which could have
+        various fields stored in entry lists.
+        """
+
+    @abstractmethod
+    def add_annotation_raw(
+        self, type_id: int, begin: int, end: int
+    ) -> Tuple[int, int]:
+        r"""This function adds an annotation entry with `begin` and `end` index
+        to the sortedlist at index `type_id` of the array which records all
+        sortedlists, return tid for the entry.
+
+        Args:
+            type_id (int): The index of Annotation sortedlist in the array.
+            begin (int): begin index of the entry.
+            end (int): end index of the entry.
+
+        Returns:
+            The `tid` of the entry.
+
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def set_attr(self, tid: int, attr_id: int, attr_value):
+        r"""This function locates the entry data with `tid` and sets its
+        attribute `attr_id` with value `attr_value`.
+
+        Args:
+            tid (int): Unique id of the entry.
+            attr_id (int): id of the attribute.
+            attr_value: value of the attribute.
+
+        """
+
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_attr(self, tid: int, attr_id: int):
+        r"""This function locates the entry data with `tid` and gets the value
+        of `attr_id` of this entry.
+
+        Args:
+            tid (int): Unique id of the entry.
+            attr_id (int): id of the attribute.
+
+        Returns:
+            The value of `attr_id` for the entry with `tid`.
+        """
+
+        raise NotImplementedError
+
+    @abstractmethod
+    def delete_entry(self, tid: int):
+        r"""This function locates the entry list with `tid` and removes it
+        from the data store.
+
+        Args:
+            tid (int): Unique id of the entry.
+
+        """
+
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_entry(self, tid: int) -> Tuple[List, int, int]:
+        r"""Look up the entry_dict with key `tid`. Find its type_id and its
+        index in the `entry_type` sortedlist.
+
+        Args:
+            tid (int): Unique id of the entry.
+
+        Returns:
+            The entry which `tid` corresponds to.
+
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def get(
+        self, entry_type: Union[str, Type[EntryType]], **kwargs
+    ) -> Iterator[List]:
+        """
+        Implementation of this method should provide to obtain the entries of
+        type `entry_type`.
+
+        Args:
+            entry_type: The type of the entry to obtain.
+
+        Returns:
+            An iterator of the entries matching the provided arguments.
+
+        """
+
+        raise NotImplementedError
+
+    @abstractmethod
+    def next_entry(self, tid: int) -> List:
+        r"""Get the next entry of the same type as the `tid` entry.
+
+        Args:
+            tid (int): Unique id of the entry.
+
+        Returns:
+            The next entry of the same type as the `tid` entry.
+
+        """
+
+        raise NotImplementedError
+
+    @abstractmethod
+    def prev_entry(self, tid: int) -> List:
+        r"""Get the previous entry of the same type as the `tid` entry.
+
+        Args:
+            tid (int): Unique id of the entry.
+
+        Returns:
+            The previous entry of the same type as the `tid` entry.
+
+        """
+
+        raise NotImplementedError