Support pyspark sql dataframe validation

.pre-commit-config.yaml

@@ -13,7 +13,7 @@ repos:
       - id: check-yaml
         description: Attempts to load all yaml files to verify syntax
       - id: debug-statements
-        description: Check for debugger imports and py37+ breakpoint() calls in python source
+        description: Check for debugger imports and py37+  calls in python source
       - id: end-of-file-fixer
         description: Makes sure files end in a newline and only a newline
       - id: trailing-whitespace

.pylintrc

@@ -1,5 +1,6 @@
 [BASIC]
-ignore=mypy.py,noxfile.py
+ignore=mypy.py,noxfile.py,pandera/accessors/pyspark_sql_accessor.py,pandera/engines/pyspark_engine.py,pandera/pyspark.py,pandera/typing/pyspark_sql.py,
+ignore-patterns=pandera/api/pyspark/*,tests/pyspark/*
 good-names=
     T,
     F,
@@ -45,4 +46,5 @@ disable=
     function-redefined,
     arguments-differ,
     unnecessary-dunder-call,
-    use-dict-literal
+    use-dict-literal,
+    invalid-name

asv_bench/benchmarks/dataframe_schema.py

@@ -2,9 +2,20 @@
 import pandas as pd
 
 from pandera import (
-    Column, DataFrameSchema, Bool, Category, Check,
-    DateTime, Float, Int, Object, String, Timedelta,
-    check_input, check_output)
+    Column,
+    DataFrameSchema,
+    Bool,
+    Category,
+    Check,
+    DateTime,
+    Float,
+    Int,
+    Object,
+    String,
+    Timedelta,
+    check_input,
+    check_output,
+)
 
 
 class Validate:
@@ -14,41 +25,46 @@ class Validate:
 
     def setup(self):
         self.schema = DataFrameSchema(
-                {
-                    "a": Column(Int),
-                    "b": Column(Float),
-                    "c": Column(String),
-                    "d": Column(Bool),
-                    "e": Column(Category),
-                    "f": Column(Object),
-                    "g": Column(DateTime),
-                    "i": Column(Timedelta),
-                    },
-                )
+            {
+                "a": Column(Int),
+                "b": Column(Float),
+                "c": Column(String),
+                "d": Column(Bool),
+                "e": Column(Category),
+                "f": Column(Object),
+                "g": Column(DateTime),
+                "i": Column(Timedelta),
+            },
+        )
         self.df = pd.DataFrame(
-                {
-                    "a": [1, 2, 3],
-                    "b": [1.1, 2.5, 9.9],
-                    "c": ["z", "y", "x"],
-                    "d": [True, True, False],
-                    "e": pd.Series(["c2", "c1", "c3"], dtype="category"),
-                    "f": [(3,), (2,), (1,)],
-                    "g": [pd.Timestamp("2015-02-01"),
-                          pd.Timestamp("2015-02-02"),
-                          pd.Timestamp("2015-02-03")],
-                    "i": [pd.Timedelta(1, unit="D"),
-                          pd.Timedelta(5, unit="D"),
-                          pd.Timedelta(9, unit="D")]
-                    })
+            {
+                "a": [1, 2, 3],
+                "b": [1.1, 2.5, 9.9],
+                "c": ["z", "y", "x"],
+                "d": [True, True, False],
+                "e": pd.Series(["c2", "c1", "c3"], dtype="category"),
+                "f": [(3,), (2,), (1,)],
+                "g": [
+                    pd.Timestamp("2015-02-01"),
+                    pd.Timestamp("2015-02-02"),
+                    pd.Timestamp("2015-02-03"),
+                ],
+                "i": [
+                    pd.Timedelta(1, unit="D"),
+                    pd.Timedelta(5, unit="D"),
+                    pd.Timedelta(9, unit="D"),
+                ],
+            }
+        )
 
     def time_df_schema(self):
         self.schema.validate(self.df)
 
     def mem_df_schema(self):
-         self.schema.validate(self.df)
+        self.schema.validate(self.df)
 
     def peakmem_df_schema(self):
-         self.schema.validate(self.df)
+        self.schema.validate(self.df)
 
 
 class Decorators:

asv_bench/benchmarks/series_schema.py

@@ -2,8 +2,20 @@
 import pandas as pd
 
 from pandera import (
-    Column, DataFrameSchema, SeriesSchema, Bool, Category, Check,
-    DateTime, Float, Int, Object, String, Timedelta, String)
+    Column,
+    DataFrameSchema,
+    SeriesSchema,
+    Bool,
+    Category,
+    Check,
+    DateTime,
+    Float,
+    Int,
+    Object,
+    String,
+    Timedelta,
+    String,
+)
 
 
 class Validate:
@@ -13,23 +25,23 @@ class Validate:
 
     def setup(self):
         self.schema = SeriesSchema(
-                String,
-                checks=[
-                    Check(lambda s: s.str.startswith("foo")),
-                    Check(lambda s: s.str.endswith("bar")),
-                    Check(lambda x: len(x) > 3, element_wise=True)
-                    ],
-                nullable=False,
-                unique=False,
-                name="my_series")
-        self.series = pd.Series(["foobar", "foobar", "foobar"],
-                                name="my_series")
+            String,
+            checks=[
+                Check(lambda s: s.str.startswith("foo")),
+                Check(lambda s: s.str.endswith("bar")),
+                Check(lambda x: len(x) > 3, element_wise=True),
+            ],
+            nullable=False,
+            unique=False,
+            name="my_series",
+        )
+        self.series = pd.Series(["foobar", "foobar", "foobar"], name="my_series")
 
     def time_series_schema(self):
         self.schema.validate(self.series)
 
     def mem_series_schema(self):
-         self.schema.validate(self.series)
+        self.schema.validate(self.series)
 
     def peakmem_series_schema(self):
-         self.schema.validate(self.series)
+        self.schema.validate(self.series)

docs/source/conf.py

@@ -188,6 +188,7 @@
 )
 copybutton_prompt_is_regexp = True
 
+
 # this is a workaround to filter out forward reference issue in
 # sphinx_autodoc_typehints
 class FilterPandasTypeAnnotationWarning(pylogging.Filter):
@@ -215,6 +216,7 @@ def filter(self, record: pylogging.LogRecord) -> bool:
     FilterPandasTypeAnnotationWarning()
 )
 
+
 # based on pandas/doc/source/conf.py
 def linkcode_resolve(domain, info):
     """Determine the URL corresponding to Python object."""

docs/source/dataframe_models.rst

@@ -205,10 +205,11 @@ You can easily convert a :class:`~pandera.api.pandas.model.DataFrameModel` class
         coerce=False,
         dtype=None,
         index=None,
-        strict=False
+        strict=False,
         name=InputSchema,
         ordered=False,
         unique_column_names=False,
+        metadata=None,
         add_missing_columns=False
     )>
 

docs/source/dataframe_schemas.rst

@@ -847,10 +847,11 @@ data pipeline:
         coerce=False,
         dtype=None,
         index=None,
-        strict=True
+        strict=True,
         name=None,
         ordered=False,
         unique_column_names=False,
+        metadata=None,
         add_missing_columns=False
     )>
 
@@ -896,10 +897,11 @@ the pipeline output.
             name=None,
             ordered=True
         )>,
-        strict=True
+        strict=True,
         name=None,
         ordered=False,
         unique_column_names=False,
+        metadata=None,
         add_missing_columns=False
     )>
 

docs/source/pyspark.rst

@@ -2,7 +2,7 @@
 
 .. _scaling_pyspark:
 
-Data Validation with Pyspark ⭐️ (New)
+Data Validation with Pyspark Pandas
 =======================================
 
 *new in 0.10.0*