minor edit

Author: MarkusRosen

preprocessing.py

@@ -1,9 +1,12 @@
+from os import remove
 import pandas as pd
 import os
 from typing import List, Dict
 from PIL import Image
 import seaborn as sns
 import matplotlib.pyplot as plt
+import numpy as np
+from torch import float32
 
 data_path = "./data/"
 
@@ -30,15 +33,42 @@ def resize_images(images: List[str]):
         new_image.save(f"{data_path}processed_images/{i}")
 
 
+def remove_outliers(df: pd.DataFrame, col: str):
+    q_low = df[col].quantile(0.02)
+    q_hi = df[col].quantile(0.98)
+    df_filtered = df[(df[col] < q_hi) & (df[col] > q_low)]
+
+    return df_filtered
+
+
 images = os.listdir(f"{data_path}images")
 df = pd.read_pickle(f"{data_path}ny_dataframe.pkl")
 
 
 df = select_rows_with_images(images, df)
 df = df.iloc[0:800]
+
+df["unformattedPrice"] = df["unformattedPrice"].astype(float)
+df["latLong_latitude"] = df["latLong_latitude"].astype(float)
+df["latLong_longitude"] = df["latLong_longitude"].astype(float)
+df["beds"] = df["beds"].astype(float)
+df["baths"] = df["baths"].astype(float)
+df["area"] = df["area"].astype(float)
 print(df.describe())
-ax = sns.boxplot(x=df["baths"])
+ax = sns.boxplot(x=df["unformattedPrice"])
 plt.show()
+print(df.dtypes)
 
+# for col in df.columns[1:]:
+#    df = remove_outliers(df, col)
+
+df = remove_outliers(df, "unformattedPrice")
+df = remove_outliers(df, "beds")
+df = remove_outliers(df, "baths")
+df = remove_outliers(df, "area")
+print(df.describe())
+ax = sns.boxplot(x=df["unformattedPrice"])
+plt.show()
 df.to_pickle(f"{data_path}df.pkl")
+
 # resize_images(images)

pytorch_lightning_regression.py

@@ -61,7 +61,7 @@ def conv_block(input_size, output_size):
 
 
 class LitClassifier(pl.LightningModule):
-    def __init__(self, lr):
+    def __init__(self, lr=1e-3):
         super().__init__()
         self.lr = lr
         self.conv1 = conv_block(3, 16)
@@ -93,7 +93,7 @@ def forward(self, x):
         return self.ln3(x)
 
     def train_dataloader(self):
-        return DataLoader(image_data, batch_size=64)
+        return DataLoader(image_data, batch_size=32)
 
     def training_step(self, batch, batch_nb):
         x, y = batch
@@ -102,32 +102,32 @@ def training_step(self, batch, batch_nb):
         # print(self(x))
         # print(y)
         # print(torch.flatten(self(x)))
-        loss = torch.nn.functional.l1_loss(torch.flatten(self(x)), y)
+        criterion = torch.nn.L1Loss()
+        y_pred = torch.flatten(self(x))
+        y_pred = y_pred.double()
+        # loss =  torch.sqrt(criterion(y_pred, y))
+        loss = criterion(y_pred, y)
 
         tensorboard_logs = {"train_loss": loss}
         return {"loss": loss, "log": tensorboard_logs}
 
     def configure_optimizers(self):
-        return torch.optim.Adam(self.parameters(), lr=(self.lr), weight_decay=1e-3 / 200)
+        return torch.optim.Adam(self.parameters(), lr=(self.lr))
 
 
 if __name__ == "__main__":
     image_data = ImageDataset(pickle_file=f"{data_path}df.pkl", image_dir=f"{data_path}processed_images/")
-    params = {"batch_size": 64}
-    # train_loader = DataLoader(image_data, **params)
 
-    model = LitClassifier(0.3)
-    mlflow_logger = pl_loggers.MLFlowLogger("logs/")
-    trainer = pl.Trainer(gpus=1, logger=mlflow_logger)
+    model = LitClassifier()
+    # mlflow_logger = pl_loggers.MLFlowLogger("logs/")
+    trainer = pl.Trainer(gpus=1)
 
     lr_finder = trainer.lr_find(model)
     print(lr_finder.results)
     fig = lr_finder.plot(suggest=True, show=True)
 
-    # fig.show(block=True)
-
     new_lr = lr_finder.suggestion()
     print(new_lr)
-    model.hparams.lr = new_lr
+    model.hparams.lr = new_lr  # 1e-2
 
     trainer.fit(model)

pytorch_regression_model.py

@@ -121,7 +121,7 @@ def train(model, device, train_loader):
     use_cuda = torch.cuda.is_available()
     device = torch.device("cuda:0" if use_cuda else "cpu")
     image_data = ImageDataset(pickle_file=f"{data_path}df.pkl", image_dir=f"{data_path}processed_images/")
-    params = {"batch_size": 4, "shuffle": True, "num_workers": 4}
+    params = {"batch_size": 64, "shuffle": True, "num_workers": 4}
     max_epochs = 200
 
     train_loader = DataLoader(image_data, **params)