Improve reader interface (#90)

* 🔨 improve reader interface

* 🔨 shrink reader code

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

* 🔥 remove redundant functionalitoes, never will use. what's the point

* 📚 updated doc string and the tutorial

* 🔨 update import statements

* 🔬 more test coverage

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

* 💚 fix unit test failure

* 📚 update reader plugin example

* 💄 update coding style

* 📚 fix index rst file

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

Co-authored-by: chfw <chfw@users.noreply.github.com>

Author: chfw

.moban.d/docs/source/index.rst renamed to .moban.d/docs/source/index.rst.jj2

@@ -6,5 +6,5 @@ targets:
   - setup.py: io_setup.py.jj2
   - .travis.yml: custom_travis.yml.jj2
   - README.rst: io_readme.rst.jj2
-  - "docs/source/index.rst": "docs/source/index.rst"
+  - "docs/source/index.rst": "docs/source/index.rst.jj2"
   - .gitignore: gitignore.jj2

.moban.yml

@@ -1,4 +1,5 @@
 
+
 5 contributors
 ================================================================================
 

CONTRIBUTORS.rst

@@ -1,3 +1,31 @@
+Extend pyexcel-io Tutorial
+================================================================================
+
+pyexcel-io itself comes with csv support.
+
+Reader
+--------------------------------------------------------------------------------
+
+Suppose we have a yaml file, containing a dictionary where the values are
+two dimensional array. The task is write reader plugin to pyexcel-io so that
+we can use get_data() to read it out.
+
+Example yaml data::
+
+.. literalinclude:: ../../examples/test.yaml
+  :language: yaml
+  
+Example code::
+
+.. literalinclude:: ../../examples/custom_yeaml_reader.py
+  :language: python
+
+
+Writer
+--------------------------------------------------------------------------------
+
+
+
 Working with xls, xlsx, and ods formats
 ================================================================================
 

docs/source/extensions.rst

@@ -3,7 +3,16 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
-{%include "header.rst.jj2" %}
+`pyexcel-io` - Let you focus on data, instead of file formats
+================================================================================
+
+:Author: chfw
+:Source code: http://github.com/pyexcel/pyexcel-io.git
+:Issues: http://github.com/pyexcel/pyexcel-io/issues
+:License: New BSD License
+:Development: |release|
+:Released: |version|
+:Generated: |today|
 
 Introduction
 --------------------------------------------------------------------------------
@@ -33,11 +42,104 @@ as of 2014. They are invented and supported by `pyexcel-io`_.
 Installation
 --------------------------------------------------------------------------------
 
-{%include "installation.rst.jj2" %}
+
+You can install pyexcel-io via pip:
+
+.. code-block:: bash
+
+    $ pip install pyexcel-io
+
+
+or clone it and install it:
+
+.. code-block:: bash
+
+    $ git clone https://github.com/pyexcel/pyexcel-io.git
+    $ cd pyexcel-io
+    $ python setup.py install
 
 For individual excel file formats, please install them as you wish:
 
-{%include "io-plugins-list.rst.jj2" %}
+.. _file-format-list:
+.. _a-map-of-plugins-and-file-formats:
+
+.. table:: A list of file formats supported by external plugins
+
+   ======================== ======================= ================= ==================
+   Package name              Supported file formats  Dependencies     Python versions
+   ======================== ======================= ================= ==================
+   `pyexcel-io`_ >=v0.6.0   csv, csvz [#f1]_, tsv,                    3.6+ 
+                            tsvz [#f2]_                               
+   `pyexcel-io`_ <=0.5.20   same as above                             2.6, 2.7, 3.3,
+                                                                      3.4, 3.5, 3.6
+                                                                      pypy
+   `pyexcel-xls`_           xls, xlsx(read only),   `xlrd`_,          same as above
+                            xlsm(read only)         `xlwt`_
+   `pyexcel-xlsx`_          xlsx                    `openpyxl`_       same as above
+   `pyexcel-ods3`_          ods                     `pyexcel-ezodf`_, 2.6, 2.7, 3.3, 3.4
+                                                    lxml              3.5, 3.6
+   `pyexcel-ods`_           ods                     `odfpy`_          same as above
+   ======================== ======================= ================= ==================
+
+.. table:: Dedicated file reader and writers
+
+   ======================== ======================= ================= ==================
+   Package name              Supported file formats  Dependencies     Python versions
+   ======================== ======================= ================= ==================
+   `pyexcel-xlsxw`_         xlsx(write only)        `XlsxWriter`_     Python 2 and 3
+   `pyexcel-xlsxr`_         xlsx(read only)         lxml              same as above
+   `pyexcel-xlsbr`_         xlsx(read only)         pyxlsb            same as above
+   `pyexcel-odsr`_          read only for ods, fods lxml              same as above
+   `pyexcel-odsw`_          write only for ods      loxun             same as above
+   `pyexcel-htmlr`_         html(read only)         lxml,html5lib     same as above
+   `pyexcel-pdfr`_          pdf(read only)          pdftables         Python 2 only.
+   ======================== ======================= ================= ==================
+
+
+Plugin shopping guide
+------------------------
+
+Except csv files, xls, xlsx and ods files are a zip of a folder containing a lot of
+xml files 
+
+The dedicated readers for excel files can stream read 
+
+
+In order to manage the list of plugins installed, you need to use pip to add or remove
+a plugin. When you use virtualenv, you can have different plugins per virtual
+environment. In the situation where you have multiple plugins that does the same thing
+in your environment, you need to tell pyexcel which plugin to use per function call.
+For example, pyexcel-ods and pyexcel-odsr, and you want to get_array to use pyexcel-odsr.
+You need to append get_array(..., library='pyexcel-odsr').
+
+
+
+.. _pyexcel-io: https://github.com/pyexcel/pyexcel-io
+.. _pyexcel-xls: https://github.com/pyexcel/pyexcel-xls
+.. _pyexcel-xlsx: https://github.com/pyexcel/pyexcel-xlsx
+.. _pyexcel-ods: https://github.com/pyexcel/pyexcel-ods
+.. _pyexcel-ods3: https://github.com/pyexcel/pyexcel-ods3
+.. _pyexcel-odsr: https://github.com/pyexcel/pyexcel-odsr
+.. _pyexcel-odsw: https://github.com/pyexcel/pyexcel-odsw
+.. _pyexcel-pdfr: https://github.com/pyexcel/pyexcel-pdfr
+
+.. _pyexcel-xlsxw: https://github.com/pyexcel/pyexcel-xlsxw
+.. _pyexcel-xlsxr: https://github.com/pyexcel/pyexcel-xlsxr
+.. _pyexcel-xlsbr: https://github.com/pyexcel/pyexcel-xlsbr
+.. _pyexcel-htmlr: https://github.com/pyexcel/pyexcel-htmlr
+
+.. _xlrd: https://github.com/python-excel/xlrd
+.. _xlwt: https://github.com/python-excel/xlwt
+.. _openpyxl: https://bitbucket.org/openpyxl/openpyxl
+.. _XlsxWriter: https://github.com/jmcnamara/XlsxWriter
+.. _pyexcel-ezodf: https://github.com/pyexcel/pyexcel-ezodf
+.. _odfpy: https://github.com/eea/odfpy
+
+
+.. rubric:: Footnotes
+
+.. [#f1] zipped csv file
+.. [#f2] zipped tsv file
 
 After that, you can start get and save data in the loaded format. There
 are two plugins for the same file format, e.g. pyexcel-ods3 and pyexcel-ods.
@@ -91,7 +193,6 @@ get_data(.., library='pyexcel-ods')
    csvz
    sqlalchemy
    django
-   options
    extensions
 
 

docs/source/index.rst

@@ -0,0 +1,45 @@
+import yaml
+from pyexcel_io import get_data
+from pyexcel_io.sheet import NamedContent
+from pyexcel_io.plugins import IOPluginInfoChainV2
+from pyexcel_io.plugin_api import ISheet, IReader
+
+
+class YourSingleSheet(ISheet):
+    def __init__(self, your_native_sheet):
+        self.two_dimensional_array = your_native_sheet
+
+    def row_iterator(self):
+        yield from self.two_dimensional_array
+
+    def column_iterator(self, row):
+        yield from row
+
+
+class YourReader(IReader):
+    def __init__(self, file_name, file_type, **keywords):
+        self.file_handle = open(file_name, "r")
+        self.native_book = yaml.load(self.file_handle)
+        self.content_array = [
+            NamedContent(key, values)
+            for key, values in self.native_book.items()
+        ]
+
+    def read_sheet(self, sheet_index):
+        two_dimensional_array = self.content_array[sheet_index].payload
+        return YourSingleSheet(two_dimensional_array)
+
+    def close(self):
+        self.file_handle.close()
+
+
+IOPluginInfoChainV2(__name__).add_a_reader(
+    relative_plugin_class_path="YourReader",
+    locations=["file"],
+    file_types=["yaml"],
+    stream_type="text",
+)
+
+if __name__ == "__main__":
+    data = get_data("test.yaml")
+    print(data)

examples/custom_yaml_reader.py

@@ -0,0 +1,11 @@
+sheet 1:
+- - 1
+  - 2
+  - 3
+- - 2
+  - 3
+  - 4
+sheet 2:
+- - A
+  - B
+  - C

examples/test.yaml

@@ -13,7 +13,7 @@ dependencies:
   - lml>=0.0.4
 test_dependencies:
   - pyexcel
-  - pyexcel-xls
+  - pyexcel-xls==0.5.9
   - SQLAlchemy
   - pyexcel-xlsxw
 extra_dependencies:

pyexcel-io.yml

@@ -51,8 +51,4 @@ def is_string(atype):
     if atype == str:
         return True
 
-    elif PY2:
-        if atype == unicode:
-            return True
-
     return False

pyexcel_io/_compact.py

@@ -7,8 +7,8 @@
     :copyright: (c) 2014-2020 by Onni Software Ltd.
     :license: New BSD License, see LICENSE for more details
 """
+from pyexcel_io.plugin_api import IReader
 from pyexcel_io.database.querysets import QuerysetsReader
-from pyexcel_io.plugin_api.abstract_reader import IReader
 
 
 class DjangoModelReader(QuerysetsReader):