Skip to content

Commit 2e7be8e

Browse files
authored
docs(examples): add pandera interoperability recipe (#169)
1 parent 82463d1 commit 2e7be8e

2 files changed

Lines changed: 66 additions & 0 deletions

File tree

examples/09_pandera_recipe.py

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
"""Recipe: validate a messy DataFrame with pandera, repair it with freshdata,
2+
then validate again to see what's fixed and what still needs manual repair.
3+
"""
4+
5+
import pandas as pd
6+
import pandera.pandas as pa
7+
8+
import freshdata as fd
9+
10+
11+
def main() -> None:
12+
df = pd.DataFrame(
13+
{
14+
"Age": [25, 41, None, 33],
15+
"Amount($)": ["$19.99", "$5.00", "$8.25", "n/a"],
16+
"Status": [" shipped ", "PENDING", "shipped", "pending"],
17+
}
18+
)
19+
20+
raw_schema = pa.DataFrameSchema(
21+
{
22+
"Age": pa.Column(float, nullable=False),
23+
"Amount($)": pa.Column(float, nullable=False),
24+
"Status": pa.Column(str, nullable=False),
25+
}
26+
)
27+
28+
print("=== Step 1: validate the raw frame ===")
29+
try:
30+
raw_schema.validate(df)
31+
except pa.errors.SchemaError as exc:
32+
print(f"Validation failed, as expected:\n{exc}\n")
33+
34+
cleaned = fd.clean(df)
35+
36+
clean_schema = pa.DataFrameSchema(
37+
{
38+
"age": pa.Column(float, nullable=False),
39+
"amount": pa.Column(float, nullable=False),
40+
"status": pa.Column(str, pa.Check.isin(["shipped", "pending"]), nullable=False),
41+
}
42+
)
43+
44+
print("=== Step 2: validate the freshdata-cleaned frame ===")
45+
try:
46+
clean_schema.validate(cleaned)
47+
print("Schema check passed after cleaning:")
48+
print(cleaned)
49+
except pa.errors.SchemaError as exc:
50+
print(f"Still failing after cleaning (freshdata renamed/typed the columns, "
51+
f"but doesn't impute nulls or normalize casing):\n{exc}\n")
52+
53+
print("=== Step 3: repair the remaining issues, then re-validate ===")
54+
repaired = cleaned.copy()
55+
repaired["age"] = repaired["age"].fillna(repaired["age"].median())
56+
repaired["amount"] = repaired["amount"].fillna(repaired["amount"].median())
57+
repaired["status"] = repaired["status"].str.lower()
58+
59+
clean_schema.validate(repaired)
60+
print("Schema check passed after repair:")
61+
print(repaired)
62+
63+
64+
if __name__ == "__main__":
65+
main()

examples/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,5 +19,6 @@ python examples/01_missing_values.py
1919
| [`06_large_dataset.py`](06_large_dataset.py) | Cleaning a large synthetic dataset, with timing |
2020
| [`07_pandas_integration.py`](07_pandas_integration.py) | Dropping freshdata into an existing pandas workflow |
2121
| [`08_csv_automation.py`](08_csv_automation.py) | Batch CSV cleaning automation with audit logs |
22+
| [`09_pandera_recipe.py`](09_pandera_recipe.py) | Validating with pandera before and after freshdata cleaning |
2223

2324
See the [documentation](https://freshcode-org.github.io/freshdata/) for full guides.

0 commit comments

Comments
 (0)