Linting

Makefile

@@ -0,0 +1,18 @@
+lint:
+	python -m pip install --quiet --upgrade pycln isort black
+	python -m pycln .
+	python -m isort .
+	python -m black .
+
+update:
+	python -m pip install --upgrade pip
+	python -m pip install --upgrade -r requirements.txt
+	python -m pip install --upgrade -r tests/requirements.txt
+
+test:
+	clear
+	python -m pytest
+
+coverage:
+	python -m coverage run -m pytest 
+	python -m coverage report --include=mabel/** -m

mabel/adapters/disk/disk_writer.py

@@ -1,8 +1,9 @@
 import os
 
+from orso.logging import get_logger
+
 from mabel.data.writers.internals.base_inner_writer import BaseInnerWriter
 from mabel.utils import paths
-from orso.logging import get_logger
 
 
 class DiskWriter(BaseInnerWriter):

mabel/adapters/google/bigquery_writer.py

@@ -1,14 +1,14 @@
 """
 BigQuery Writer
 """
-import os
 from enum import Enum
 from typing import Optional
 
 from google.cloud.exceptions import NotFound
+from orso.logging.create_logger import get_logger
+
 from mabel.data.writers.internals.base_inner_writer import BaseInnerWriter
 from mabel.errors import MissingDependencyError
-from orso.logging.create_logger import get_logger
 
 try:
     from google.cloud import bigquery  # type:ignore

mabel/adapters/google/google_cloud_storage_writer.py

@@ -1,10 +1,11 @@
 import os
 
-from mabel.data.writers.internals.base_inner_writer import BaseInnerWriter
-from mabel.errors import MissingDependencyError
 from orso.logging.create_logger import get_logger
 from urllib3.exceptions import ProtocolError  # type:ignore
 
+from mabel.data.writers.internals.base_inner_writer import BaseInnerWriter
+from mabel.errors import MissingDependencyError
+
 try:
     from google.api_core import retry  # type:ignore
     from google.api_core.exceptions import InternalServerError  # type:ignore

mabel/adapters/minio/minio_reader.py

@@ -1,7 +1,6 @@
 """
 MinIo Reader - also works with AWS
 """
-from functools import lru_cache
 
 from ...data.readers.internals.base_inner_reader import BaseInnerReader
 from ...errors import MissingDependencyError

mabel/adapters/sql/__init__.py

@@ -0,0 +1 @@
+from .sql_writer import SqlWriter

mabel/adapters/sql/sql_writer.py

@@ -0,0 +1,18 @@
+from mabel.data.writers.internals.base_inner_writer import BaseInnerWriter
+
+
+class SqlWriter(BaseInnerWriter):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        kwargs_passed = [f"{k}={v!r}" for k, v in kwargs.items()]
+        self.formatted_args = ", ".join(kwargs_passed)
+
+    def commit(self, byte_data, override_blob_name=None):
+        # if we've been given the filename, use that, otherwise get the
+        # name from the path builder
+        if override_blob_name:  # prgama: no cover
+            blob_name = override_blob_name
+        else:
+            blob_name = self._build_path()
+
+        return f"NullWriter({blob_name})"

mabel/data/internals/dictset.py

@@ -33,6 +33,8 @@
 from typing import Union
 
 import orjson
+from orso.cityhash import CityHash64
+
 from mabel.data.internals.display import ascii_table
 from mabel.data.internals.display import html_table
 from mabel.data.internals.dnf_filters import DnfFilters
@@ -44,7 +46,6 @@
 from mabel.data.internals.storage_classes import StorageClassMemory
 from mabel.errors import MissingDependencyError
 from mabel.utils.ipython import is_running_from_ipython
-from orso.cityhash import CityHash64
 
 
 class STORAGE_CLASS(int, Enum):

mabel/data/readers/internals/base_inner_reader.py

@@ -8,9 +8,10 @@
 from io import IOBase
 from typing import Iterable
 
+from orso.logging import get_logger
+
 from mabel.utils import dates
 from mabel.utils import paths
-from orso.logging import get_logger
 
 BUFFER_SIZE: int = 64 * 1024 * 1024  # 64Mb
 

mabel/data/readers/internals/inline_functions.py

@@ -9,9 +9,10 @@
 from math import trunc
 
 import orjson
-from mabel.utils.dates import parse_iso
 from orso.cityhash import CityHash64
 
+from mabel.utils.dates import parse_iso
+
 
 def get_year(input):
     """

mabel/data/readers/internals/parallel_reader.py

@@ -24,12 +24,13 @@
 from enum import Enum
 from functools import reduce
 
+from orso import logging
+
 from mabel.data.internals.dnf_filters import DnfFilters
 from mabel.data.internals.expression import Expression
 from mabel.data.internals.index import Index
 from mabel.data.internals.records import flatten
 from mabel.utils import paths
-from orso import logging
 
 from . import decompressors
 from . import parsers

mabel/data/readers/internals/sql_reader.py

@@ -19,11 +19,12 @@
 import re
 from typing import Optional
 
+from orso.logging import get_logger
+
 from mabel.data.readers.internals.inline_evaluator import Evaluator
 from mabel.data.readers.internals.inline_evaluator import get_function_name
 from mabel.utils.token_labeler import TOKENS
 from mabel.utils.token_labeler import Tokenizer
-from orso.logging import get_logger
 
 SQL_PARTS = [
     r"SELECT",

mabel/data/readers/reader.py

@@ -3,6 +3,8 @@
 from typing import Optional
 from typing import Union
 
+from orso.logging import get_logger
+
 from mabel.data.internals.dictset import STORAGE_CLASS
 from mabel.data.internals.dictset import DictSet
 from mabel.data.internals.dnf_filters import DnfFilters
@@ -18,7 +20,6 @@
 from mabel.errors import InvalidCombinationError
 from mabel.utils.dates import parse_delta
 from mabel.utils.parameter_validator import validate
-from orso.logging import get_logger
 
 # fmt:off
 RULES = [

mabel/data/validator/schema.py

@@ -5,9 +5,10 @@
 from typing import Union
 
 import orjson
-from mabel.errors import ValidationError
 from orso.schema import RelationSchema
 
+from mabel.errors import ValidationError
+
 
 class Schema:
     def __init__(self, definition: Union[str, List[Dict[str, Any]], dict]):

mabel/data/writers/batch_writer.py

@@ -3,9 +3,10 @@
 from typing import Any
 
 import orjson
-from mabel.utils import paths
 from orso.logging import get_logger
 
+from mabel.utils import paths
+
 from .internals.blob_writer import BlobWriter
 from .writer import Writer
 

mabel/data/writers/internals/blob_writer copy.py

@@ -5,10 +5,11 @@
 
 import orjson
 import zstandard
+from orso.logging import get_logger
+
 from mabel.data.internals.records import flatten
 from mabel.data.validator import Schema
 from mabel.errors import MissingDependencyError
-from orso.logging import get_logger
 
 BLOB_SIZE = 64 * 1024 * 1024  # 64Mb, 16 files per gigabyte
 SUPPORTED_FORMATS_ALGORITHMS = ("jsonl", "zstd", "parquet", "text", "flat")

mabel/data/writers/internals/blob_writer.py

@@ -5,10 +5,11 @@
 
 import orjson
 import zstandard
+from orso.logging import get_logger
+
 from mabel.data.internals.records import flatten
 from mabel.data.validator import Schema
 from mabel.errors import MissingDependencyError
-from orso.logging import get_logger
 
 BLOB_SIZE = 64 * 1024 * 1024  # 64Mb, 16 files per gigabyte
 SUPPORTED_FORMATS_ALGORITHMS = ("jsonl", "zstd", "parquet", "text", "flat")

mabel/data/writers/stream_writer.py

@@ -3,12 +3,12 @@
 import re
 import threading
 import time
-from typing import Union
+
+from orso.logging import get_logger
 
 from mabel.utils import dates
 from mabel.utils import paths
 from mabel.utils import text
-from orso.logging import get_logger
 
 from .internals.writer_pool import WriterPool
 from .writer import Writer

mabel/data/writers/writer.py

@@ -4,14 +4,15 @@
 from typing import Union
 
 import orjson
+from orso.logging import get_logger
+
 from mabel.data.validator import Schema
 from mabel.data.writers.internals.blob_writer import BlobWriter
 from mabel.errors import InvalidDataSetError
 from mabel.errors import MissingDependencyError
 from mabel.errors import ValidationError
 from mabel.utils import dates
 from mabel.utils import paths
-from orso.logging import get_logger
 
 logger = get_logger()
 

mabel/utils/parameter_validator.py

@@ -1,9 +1,10 @@
 import functools
 import inspect
 
-from mabel.errors import InvalidReaderConfigError
 from orso.logging import get_logger
 
+from mabel.errors import InvalidReaderConfigError
+
 
 def get_levenshtein_distance(word1, word2):
     """

mabel/version.py

@@ -1,6 +1,6 @@
 # Store the version here so:
 # 1) we don't load dependencies by storing it in __init__.py
 # 2) we can import it in setup.py for the same reason
-__version__ = "0.6.14"
+__version__ = "0.6.15"
 
 # nodoc - don't add to the documentation wiki

pyproject.toml

@@ -1,13 +1,18 @@
 [tool.black]
 line-length = 100
-target-version = ["py310"]
+target-version = ['py310']
+fast = true
+jobs = 4
+cache = true
 
 [tool.isort]
 profile = "black"
-src_paths = ["mabel/mabel"]
-extend_skip_glob = ["tests/**", "*.pyx"]
+extend_skip_glob = ["tests/**", "*.pyx", "testdata/**"]
+skip_gitignore = true
 line_length = 100
 multi_line_output = 9
 force_single_line = true
 float_to_top = true
 
+[tool.pylint.messages_control]
+disable = "C0103,C0415"

requirements.txt

@@ -1,4 +1,3 @@
-bitarray
 lz4
 orjson
 orso

tests/performance/arrow.py

@@ -1,7 +1,6 @@
 from pyarrow import json
 import pyarrow.parquet as pq
 import pyarrow as pa
-from glob import glob
 
 import time
 

tests/performance/avro_performance.py

@@ -11,7 +11,6 @@
 import avro.schema
 from avro.datafile import DataFileReader, DataFileWriter
 from avro.io import DatumReader, DatumWriter
-import datetime
 import os
 import sys
 

tests/performance/dictset_where_performance.py

@@ -3,7 +3,6 @@
 #
 #   old dictset : 6.69
 #   filter      : 5.29
-import datetime
 import time
 import os
 import sys

tests/performance/indexing.py

@@ -3,8 +3,6 @@
 """
 import sys
 import os
-import time
-import statistics
 
 sys.path.insert(1, os.path.join(sys.path[0], "../.."))
 from mabel.adapters.null import NullWriter

tests/performance/json_performance.py

@@ -86,14 +86,11 @@ def time_it_4():
     return (time.perf_counter_ns() - start) / 1e9
 
 
-import json
 import ujson
 import orjson
 
 # import cysimdjson
 import simdjson
-import os
-import sys
 
 simparser = simdjson.Parser()
 

tests/performance/validator_performance.py

@@ -6,14 +6,12 @@
 │ pydantic │ 2.1748749800000002 │
 └──────────┴────────────────────┘
 """
-import cProfile
 import time
 import pydantic
 import datetime
 import statistics
 from typing import Optional
 from pydantic import create_model
-import shutil
 import sys
 import os
 

tests/test_utils_config.py

@@ -1,4 +1,3 @@
-import datetime
 import sys
 import os
 import pytest