One reader/writer handle many file types, i.e. csv reader for csv and tsv (#81)

chfw · web-flow · commit 020b8f0a9fa4 · 2020-09-27T23:30:53.000+01:00
* 🐛 pyexcel-io used to allow one reader/writer handles more than one file type

* 📚 update pyinstaller instructions

* This is an auto-commit, updating project meta data, such as changelog.rst, contributors.rst

* 💄 update coding style

Co-authored-by: chfw &lt;chfw@users.noreply.github.com&gt;
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -16,7 +16,7 @@ Change log
 **updated**
 
 #. pyexcel-io plugin interface has been rewritten. PyInstaller user will be
-   impacted.
+   impacted. please read 'Packaging with Pyinstaller' in the documentation.
 
 0.5.20 - 17.7.2019
 --------------------------------------------------------------------------------
diff --git a/changelog.yml b/changelog.yml
@@ -10,7 +10,8 @@ releases:
     - 'python 3.6 lower versions are no longer supported'
   - action: updated
     details:
-    - 'pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted.'
+    - pyexcel-io plugin interface has been rewritten. PyInstaller user will be impacted.
+      please read 'Packaging with Pyinstaller' in the documentation.
   version: 0.6.0
   date: tbd
 - changes:
diff --git a/docs/source/pyinstaller.rst b/docs/source/pyinstaller.rst
@@ -13,14 +13,9 @@ In order to package every built-in plugins of pyexcel-io, you need to specify::
     --hidden-import pyexcel_io.readers.csv_in_memory
     --hidden-import pyexcel_io.readers.csv_content
     --hidden-import pyexcel_io.readers.csvz
-    --hidden-import pyexcel_io.readers.tsv
-    --hidden-import pyexcel_io.readers.tsvz
     --hidden-import pyexcel_io.writers.csv_in_file
     --hidden-import pyexcel_io.writers.csv_in_memory
-    --hidden-import pyexcel_io.writers.tsv_in_file
-    --hidden-import pyexcel_io.writers.tsv_in_memory
     --hidden-import pyexcel_io.writers.csvz_writer
-    --hidden-import pyexcel_io.writers.tsvz_writer
     --hidden-import pyexcel_io.database.importers.django
     --hidden-import pyexcel_io.database.importers.sqlalchemy
     --hidden-import pyexcel_io.database.exporters.django
diff --git a/pyexcel_io/database/exporters/django.py b/pyexcel_io/database/exporters/django.py
@@ -29,7 +29,7 @@ def __init__(self, model, export_columns=None, **keywords):
 class DjangoBookReader(object):
     """ read django models """
 
-    def __init__(self, exporter, **keywords):
+    def __init__(self, exporter, _, **keywords):
         self.exporter = exporter
         self.keywords = keywords
         self.content_array = self.exporter.adapters
diff --git a/pyexcel_io/database/exporters/sqlalchemy.py b/pyexcel_io/database/exporters/sqlalchemy.py
@@ -33,7 +33,7 @@ def __init__(self, session, table, export_columns=None, **keywords):
 class SQLBookReader(object):
     """ read a table via sqlalchemy """
 
-    def __init__(self, exporter, **keywords):
+    def __init__(self, exporter, _, **keywords):
         self.__exporter = exporter
         self.content_array = self.__exporter.adapters
         self.keywords = keywords
diff --git a/pyexcel_io/database/importers/django.py b/pyexcel_io/database/importers/django.py
@@ -60,7 +60,7 @@ def close(self):
 class DjangoBookWriter(object):
     """ write data into django models """
 
-    def __init__(self, exporter, **keywords):
+    def __init__(self, exporter, _, **keywords):
         self.__importer = exporter
         self._keywords = keywords
 
diff --git a/pyexcel_io/database/importers/sqlalchemy.py b/pyexcel_io/database/importers/sqlalchemy.py
@@ -74,7 +74,7 @@ def close(self):
 class SQLBookWriter(object):
     """ write data into database tables via sqlalchemy """
 
-    def __init__(self, file_content, auto_commit=True, **keywords):
+    def __init__(self, file_content, _, auto_commit=True, **keywords):
         self.__importer = file_content
         self.__auto_commit = auto_commit
 
diff --git a/pyexcel_io/reader.py b/pyexcel_io/reader.py
@@ -37,7 +37,9 @@ def open(self, file_name, **keywords):
             self.file_type, location="file", library=self.library
         )
         self.keywords, native_sheet_keywords = clean_keywords(keywords)
-        self.reader = reader_class(file_name, **native_sheet_keywords)
+        self.reader = reader_class(
+            file_name, self.file_type, **native_sheet_keywords
+        )
         return self.reader
 
     def open_content(self, file_content, **keywords):
@@ -46,7 +48,9 @@ def open_content(self, file_content, **keywords):
             reader_class = NEW_READERS.get_a_plugin(
                 self.file_type, location="content", library=self.library
             )
-            self.reader = reader_class(file_content, **native_sheet_keywords)
+            self.reader = reader_class(
+                file_content, self.file_type, **native_sheet_keywords
+            )
             return self.reader
         except (
             exceptions.NoSupportingPluginFound,
@@ -62,7 +66,9 @@ def open_stream(self, file_stream, **keywords):
         reader_class = NEW_READERS.get_a_plugin(
             self.file_type, location="memory", library=self.library
         )
-        self.reader = reader_class(file_stream, **native_sheet_keywords)
+        self.reader = reader_class(
+            file_stream, self.file_type, **native_sheet_keywords
+        )
         return self.reader
 
     def read_sheet_by_name(self, sheet_name):
diff --git a/pyexcel_io/readers/__init__.py b/pyexcel_io/readers/__init__.py
@@ -12,41 +12,21 @@
 NewIOPluginInfoChain(__name__).add_a_reader(
     relative_plugin_class_path="csv_in_file.FileReader",
     locations=["file"],
-    file_types=["csv"],
+    file_types=["csv", "tsv"],
     stream_type="text",
 ).add_a_reader(
     relative_plugin_class_path="csv_content.ContentReader",
     locations=["content"],
-    file_types=["csv"],
+    file_types=["csv", "tsv"],
     stream_type="text",
 ).add_a_reader(
     relative_plugin_class_path="csv_in_memory.MemoryReader",
     locations=["memory"],
-    file_types=["csv"],
-    stream_type="text",
-).add_a_reader(
-    relative_plugin_class_path="tsv.TSVMemoryReader",
-    locations=["memory"],
-    file_types=["tsv"],
-    stream_type="text",
-).add_a_reader(
-    relative_plugin_class_path="tsv.TSVFileReader",
-    locations=["file"],
-    file_types=["tsv"],
-    stream_type="text",
-).add_a_reader(
-    relative_plugin_class_path="tsv.TSVContentReader",
-    locations=["content"],
-    file_types=["tsv"],
+    file_types=["csv", "tsv"],
     stream_type="text",
 ).add_a_reader(
     relative_plugin_class_path="csvz.FileReader",
-    file_types=["csvz"],
-    locations=["file", "memory"],
-    stream_type="binary",
-).add_a_reader(
-    relative_plugin_class_path="tsvz.TSVZipFileReader",
-    file_types=["tsvz"],
+    file_types=["csvz", "tsvz"],
     locations=["file", "memory"],
     stream_type="binary",
 )
diff --git a/pyexcel_io/readers/csv_content.py b/pyexcel_io/readers/csv_content.py
@@ -1,19 +1,16 @@
 import mmap
 
-import pyexcel_io.constants as constants
 from pyexcel_io.book import _convert_content_to_stream
 from pyexcel_io.readers.csv_sheet import CSVMemoryMapIterator
 from pyexcel_io.readers.csv_in_memory import MemoryReader
 
 
 class ContentReader(MemoryReader):
-    file_type = constants.FILE_FORMAT_CSV
-
-    def __init__(self, file_content, **keywords):
+    def __init__(self, file_content, file_type, **keywords):
         file_stream = ContentReader.convert_content_to_stream(
-            file_content, self.file_type, **keywords
+            file_content, file_type, **keywords
         )
-        super().__init__(file_stream, **keywords)
+        super().__init__(file_stream, file_type, **keywords)
 
     @staticmethod
     def convert_content_to_stream(file_content, file_type, **keywords):
diff --git a/pyexcel_io/readers/csv_in_file.py b/pyexcel_io/readers/csv_in_file.py
@@ -10,13 +10,15 @@
 
 
 class FileReader(object):
-    def __init__(self, file_name, **keywords):
+    def __init__(self, file_name, file_type, **keywords):
         """Load content from a file
         :params str filename: an accessible file path
         :returns: a book
         """
         self.handles = []
         self.keywords = keywords
+        if file_type == constants.FILE_FORMAT_TSV:
+            self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
         self.__line_terminator = keywords.get(
             constants.KEYWORD_LINE_TERMINATOR, DEFAULT_NEWLINE
         )
diff --git a/pyexcel_io/readers/csv_in_memory.py b/pyexcel_io/readers/csv_in_memory.py
@@ -9,15 +9,19 @@
 
 
 class MemoryReader(object):
-    file_type = constants.FILE_FORMAT_CSV
-
-    def __init__(self, file_stream, multiple_sheets=False, **keywords):
+    def __init__(
+        self, file_stream, file_type, multiple_sheets=False, **keywords
+    ):
         """Load content from memory
         :params stream file_content: the actual file content in memory
         :returns: a book
         """
         self.handles = []
         self.keywords = keywords
+        if file_type == constants.FILE_FORMAT_TSV:
+            self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
+        self.file_type = file_type
+
         self.__load_from_memory_flag = True
         self.__line_terminator = keywords.get(
             constants.KEYWORD_LINE_TERMINATOR, constants.DEFAULT_CSV_NEWLINE
diff --git a/pyexcel_io/readers/csvz.py b/pyexcel_io/readers/csvz.py
@@ -10,13 +10,14 @@
 import zipfile
 
 import chardet
+from pyexcel_io import constants
 from pyexcel_io.sheet import NamedContent
 from pyexcel_io._compact import StringIO
 from pyexcel_io.readers.csv_sheet import CSVinMemoryReader
 
 
 class FileReader(object):
-    def __init__(self, file_alike_object, **keywords):
+    def __init__(self, file_alike_object, file_type, **keywords):
         self.content_array = []
         try:
             self.zipfile = zipfile.ZipFile(file_alike_object, "r")
@@ -26,6 +27,8 @@ def __init__(self, file_alike_object, **keywords):
             ]
             self.content_array = sheets
             self.keywords = keywords
+            if file_type == constants.FILE_FORMAT_TSVZ:
+                self.keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
 
         except zipfile.BadZipfile:
             print("StringIO instance was passed by any chance?")
diff --git a/pyexcel_io/readers/tsv.py b/pyexcel_io/readers/tsv.py
diff --git a/pyexcel_io/readers/tsvz.py b/pyexcel_io/readers/tsvz.py
diff --git a/pyexcel_io/writer.py b/pyexcel_io/writer.py
@@ -11,21 +11,21 @@ def open(self, file_name, **keywords):
         writer_class = NEW_WRITERS.get_a_plugin(
             self.file_type, library=self.library, location="file"
         )
-        self.writer = writer_class(file_name, **keywords)
+        self.writer = writer_class(file_name, self.file_type, **keywords)
 
     def open_content(self, file_stream, **keywords):
         # if not isstream(file_stream):
         #    raise IOError(MESSAGE_ERROR_03)
         writer_class = NEW_WRITERS.get_a_plugin(
             self.file_type, library=self.library, location="content"
         )
-        self.writer = writer_class(file_stream, **keywords)
+        self.writer = writer_class(file_stream, self.file_type, **keywords)
 
     def open_stream(self, file_stream, **keywords):
         writer_class = NEW_WRITERS.get_a_plugin(
             self.file_type, library=self.library, location="memory"
         )
-        self.writer = writer_class(file_stream, **keywords)
+        self.writer = writer_class(file_stream, self.file_type, **keywords)
 
     def write(self, incoming_dict):
         self.writer.write(incoming_dict)
diff --git a/pyexcel_io/writers/__init__.py b/pyexcel_io/writers/__init__.py
@@ -12,31 +12,16 @@
 NewIOPluginInfoChain(__name__).add_a_writer(
     relative_plugin_class_path="csv_in_file.CsvFileWriter",
     locations=["file", "content"],
-    file_types=["csv"],
+    file_types=["csv", "tsv"],
     stream_type="text",
 ).add_a_writer(
     relative_plugin_class_path="csv_in_memory.CsvMemoryWriter",
     locations=["memory"],
-    file_types=["csv"],
-    stream_type="text",
-).add_a_writer(
-    relative_plugin_class_path="tsv_in_file.TsvFileWriter",
-    locations=["file", "content"],
-    file_types=["tsv"],
-    stream_type="text",
-).add_a_writer(
-    relative_plugin_class_path="tsv_in_memory.TsvMemoryWriter",
-    locations=["memory"],
-    file_types=["tsv"],
+    file_types=["csv", "tsv"],
     stream_type="text",
 ).add_a_writer(
     relative_plugin_class_path="csvz_writer.CsvZipWriter",
     locations=["memory", "file", "content"],
-    file_types=["csvz"],
-    stream_type="binary",
-).add_a_writer(
-    relative_plugin_class_path="tsvz_writer.TsvZipWriter",
-    locations=["memory", "file", "content"],
-    file_types=["tsvz"],
+    file_types=["csvz", "tsvz"],
     stream_type="binary",
 )
diff --git a/pyexcel_io/writers/csv_in_file.py b/pyexcel_io/writers/csv_in_file.py
@@ -1,10 +1,13 @@
+from pyexcel_io import constants
 from pyexcel_io.writers.csv_sheet import CSVFileWriter
 
 
 class CsvFileWriter:
-    def __init__(self, file_alike_object, **keywords):
+    def __init__(self, file_alike_object, file_type, **keywords):
         self._file_alike_object = file_alike_object
         self._keywords = keywords
+        if file_type == constants.FILE_FORMAT_TSV:
+            self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
         self.__index = 0
         self.writer = None
 
diff --git a/pyexcel_io/writers/csv_in_memory.py b/pyexcel_io/writers/csv_in_memory.py
@@ -1,10 +1,13 @@
+from pyexcel_io import constants
 from pyexcel_io.writers.csv_sheet import CSVMemoryWriter
 
 
 class CsvMemoryWriter:
-    def __init__(self, file_alike_object, **keywords):
+    def __init__(self, file_alike_object, file_type, **keywords):
         self._file_alike_object = file_alike_object
         self._keywords = keywords
+        if file_type == constants.FILE_FORMAT_TSV:
+            self._keywords["dialect"] = constants.KEYWORD_TSV_DIALECT
         self.__index = 0
 
     def create_sheet(self, name):
diff --git a/pyexcel_io/writers/csvz_writer.py b/pyexcel_io/writers/csvz_writer.py
diff --git a/pyexcel_io/writers/tsv_in_file.py b/pyexcel_io/writers/tsv_in_file.py
diff --git a/pyexcel_io/writers/tsv_in_memory.py b/pyexcel_io/writers/tsv_in_memory.py
diff --git a/pyexcel_io/writers/tsvz_writer.py b/pyexcel_io/writers/tsvz_writer.py
diff --git a/tests/test_django_book.py b/tests/test_django_book.py
diff --git a/tests/test_sql_book.py b/tests/test_sql_book.py
