|
| 1 | +"""Recipe: validate a messy DataFrame with pandera, repair it with freshdata, |
| 2 | +then validate again to see what's fixed and what still needs manual repair. |
| 3 | +""" |
| 4 | + |
| 5 | +import pandas as pd |
| 6 | +import pandera.pandas as pa |
| 7 | + |
| 8 | +import freshdata as fd |
| 9 | + |
| 10 | + |
| 11 | +def main() -> None: |
| 12 | + df = pd.DataFrame( |
| 13 | + { |
| 14 | + "Age": [25, 41, None, 33], |
| 15 | + "Amount($)": ["$19.99", "$5.00", "$8.25", "n/a"], |
| 16 | + "Status": [" shipped ", "PENDING", "shipped", "pending"], |
| 17 | + } |
| 18 | + ) |
| 19 | + |
| 20 | + raw_schema = pa.DataFrameSchema( |
| 21 | + { |
| 22 | + "Age": pa.Column(float, nullable=False), |
| 23 | + "Amount($)": pa.Column(float, nullable=False), |
| 24 | + "Status": pa.Column(str, nullable=False), |
| 25 | + } |
| 26 | + ) |
| 27 | + |
| 28 | + print("=== Step 1: validate the raw frame ===") |
| 29 | + try: |
| 30 | + raw_schema.validate(df) |
| 31 | + except pa.errors.SchemaError as exc: |
| 32 | + print(f"Validation failed, as expected:\n{exc}\n") |
| 33 | + |
| 34 | + cleaned = fd.clean(df) |
| 35 | + |
| 36 | + clean_schema = pa.DataFrameSchema( |
| 37 | + { |
| 38 | + "age": pa.Column(float, nullable=False), |
| 39 | + "amount": pa.Column(float, nullable=False), |
| 40 | + "status": pa.Column(str, pa.Check.isin(["shipped", "pending"]), nullable=False), |
| 41 | + } |
| 42 | + ) |
| 43 | + |
| 44 | + print("=== Step 2: validate the freshdata-cleaned frame ===") |
| 45 | + try: |
| 46 | + clean_schema.validate(cleaned) |
| 47 | + print("Schema check passed after cleaning:") |
| 48 | + print(cleaned) |
| 49 | + except pa.errors.SchemaError as exc: |
| 50 | + print(f"Still failing after cleaning (freshdata renamed/typed the columns, " |
| 51 | + f"but doesn't impute nulls or normalize casing):\n{exc}\n") |
| 52 | + |
| 53 | + print("=== Step 3: repair the remaining issues, then re-validate ===") |
| 54 | + repaired = cleaned.copy() |
| 55 | + repaired["age"] = repaired["age"].fillna(repaired["age"].median()) |
| 56 | + repaired["amount"] = repaired["amount"].fillna(repaired["amount"].median()) |
| 57 | + repaired["status"] = repaired["status"].str.lower() |
| 58 | + |
| 59 | + clean_schema.validate(repaired) |
| 60 | + print("Schema check passed after repair:") |
| 61 | + print(repaired) |
| 62 | + |
| 63 | + |
| 64 | +if __name__ == "__main__": |
| 65 | + main() |
0 commit comments