added early stopping, validation and testing

Author: MarkusRosen

.gitignore

@@ -5,3 +5,7 @@
 /mlruns
 /default
 /1
+/multi_input
+/my_model
+/image_only
+/temp_model.ckpt

boxplot.png

@@ -43,32 +43,40 @@ def remove_outliers(df: pd.DataFrame, col: str):
 
 images = os.listdir(f"{data_path}images")
 df = pd.read_pickle(f"{data_path}ny_dataframe.pkl")
+# print(images)
 
 
 df = select_rows_with_images(images, df)
-df = df.iloc[0:800]
+# df = df.iloc[0:800]
 
 df["unformattedPrice"] = df["unformattedPrice"].astype(float)
 df["latLong_latitude"] = df["latLong_latitude"].astype(float)
 df["latLong_longitude"] = df["latLong_longitude"].astype(float)
 df["beds"] = df["beds"].astype(float)
 df["baths"] = df["baths"].astype(float)
 df["area"] = df["area"].astype(float)
+
+# df["unformattedPrice"] = df["unformattedPrice"]/df["area"]
+df.columns = ["zpid", "price", "latitude", "longitude", "beds", "baths", "area"]
 print(df.describe())
-ax = sns.boxplot(x=df["unformattedPrice"])
-plt.show()
+ax = sns.boxplot(x=df["price"])
+# plt.show()
 print(df.dtypes)
 
 # for col in df.columns[1:]:
 #    df = remove_outliers(df, col)
 
-df = remove_outliers(df, "unformattedPrice")
+df = remove_outliers(df, "price")
 df = remove_outliers(df, "beds")
 df = remove_outliers(df, "baths")
 df = remove_outliers(df, "area")
+# dataset has to be divisble by 0.8!
+
+df = df.iloc[3:]
 print(df.describe())
-ax = sns.boxplot(x=df["unformattedPrice"])
-plt.show()
+ax = sns.boxplot(x=df["price"])
+# plt.show()
+print(df)
 df.to_pickle(f"{data_path}df.pkl")
-
-# resize_images(images)
+# print(list(df["zpid"] + ".png"))
+# resize_images(list(df["zpid"] + ".png"))

lr_i.png

@@ -1,18 +1,16 @@
+import pandas as pd
+import numpy as np
+from PIL import Image
+
 import torch
 import torch.nn as nn
-import torch.nn.functional as F
 from torch.utils.data import Dataset, DataLoader, random_split
 from torchvision import transforms
-from PIL import Image
-import os
-import pandas as pd
-import numpy as np
-from tqdm import tqdm
-from datetime import datetime
 
 import pytorch_lightning as pl
-from pytorch_lightning import loggers as pl_loggers
-import matplotlib.pyplot as plt
+from pytorch_lightning.logging import TensorBoardLogger
+from pytorch_lightning.callbacks.early_stopping import EarlyStopping
+
 
 data_path = "./data/"
 
@@ -23,11 +21,8 @@ class ImageDataset(Dataset):
     def __init__(self, pickle_file, image_dir):
         self.image_dir = image_dir
         self.pickle_file = pickle_file
-
         self.tabular = pd.read_pickle(pickle_file)
 
-        print(self.tabular)
-
     def __len__(self):
         return len(self.tabular)
 
@@ -37,19 +32,19 @@ def __getitem__(self, idx):
 
         tabular = self.tabular.iloc[idx, 0:]
 
-        y = tabular["unformattedPrice"]
+        y = tabular["price"]
 
         image = Image.open(f"{self.image_dir}/{tabular['zpid']}.png")
         image = np.array(image)
         image = image[..., :3]
 
         image = transforms.functional.to_tensor(image)
 
-        tabular = tabular[["latLong_latitude", "latLong_longitude", "beds", "baths", "area"]]
+        tabular = tabular[["latitude", "longitude", "beds", "baths", "area"]]
         tabular = tabular.tolist()
         tabular = torch.FloatTensor(tabular)
 
-        return image, y
+        return image, tabular, y
 
 
 def conv_block(input_size, output_size):
@@ -61,73 +56,133 @@ def conv_block(input_size, output_size):
 
 
 class LitClassifier(pl.LightningModule):
-    def __init__(self, lr=1e-3):
+    def __init__(
+        self, lr: float = 1e-3, num_workers: int = 4, batch_size: int = 32,
+    ):
         super().__init__()
         self.lr = lr
+        self.num_workers = num_workers
+        self.batch_size = batch_size
+
         self.conv1 = conv_block(3, 16)
         self.conv2 = conv_block(16, 32)
         self.conv3 = conv_block(32, 64)
-        # conv2d -> -2 pixels
-        # max pool -> pixels/2
-        # remainder will be dropped
+
         self.ln1 = nn.Linear(64 * 26 * 26, 16)
         self.relu = nn.ReLU()
         self.batchnorm = nn.BatchNorm1d(16)
         self.dropout = nn.Dropout2d(0.5)
-        self.ln2 = nn.Linear(16, 4)
-        self.ln3 = nn.Linear(4, 1)
-
-    def forward(self, x):
-        x = self.conv1(x)
-        x = self.conv2(x)
-        x = self.conv3(x)
-        x = x.reshape(x.shape[0], -1)
-        x = self.ln1(x)
-        x = self.relu(x)
-        x = self.batchnorm(x)
-        x = self.dropout(x)
-        x = self.ln2(x)
+        self.ln2 = nn.Linear(16, 5)
+
+        self.ln4 = nn.Linear(5, 10)
+        self.ln5 = nn.Linear(10, 10)
+        self.ln6 = nn.Linear(10, 5)
+        self.ln7 = nn.Linear(10, 1)
+
+    def forward(self, img, tab):
+        img = self.conv1(img)
+
+        img = self.conv2(img)
+        img = self.conv3(img)
+        img = img.reshape(img.shape[0], -1)
+        img = self.ln1(img)
+        img = self.relu(img)
+        img = self.batchnorm(img)
+        img = self.dropout(img)
+        img = self.ln2(img)
+        img = self.relu(img)
+
+        tab = self.ln4(tab)
+        tab = self.relu(tab)
+        tab = self.ln5(tab)
+        tab = self.relu(tab)
+        tab = self.ln6(tab)
+        tab = self.relu(tab)
+
+        x = torch.cat((img, tab), dim=1)
         x = self.relu(x)
-        # x = self.ln3(x)
-        # print(x)
-        return self.ln3(x)
 
-    def train_dataloader(self):
-        return DataLoader(image_data, batch_size=32)
-
-    def training_step(self, batch, batch_nb):
-        x, y = batch
-        # print(x)
-        # print(y)
-        # print(self(x))
-        # print(y)
-        # print(torch.flatten(self(x)))
+        return self.ln7(x)
+
+    def training_step(self, batch, batch_idx):
+        image, tabular, y = batch
+
         criterion = torch.nn.L1Loss()
-        y_pred = torch.flatten(self(x))
+        y_pred = torch.flatten(self(image, tabular))
         y_pred = y_pred.double()
-        # loss =  torch.sqrt(criterion(y_pred, y))
+
         loss = criterion(y_pred, y)
 
         tensorboard_logs = {"train_loss": loss}
         return {"loss": loss, "log": tensorboard_logs}
 
+    def validation_step(self, batch, batch_idx):
+        image, tabular, y = batch
+
+        criterion = torch.nn.L1Loss()
+        y_pred = torch.flatten(self(image, tabular))
+        y_pred = y_pred.double()
+
+        val_loss = criterion(y_pred, y)
+
+        return {"val_loss": val_loss}
+
+    def validation_epoch_end(self, outputs):
+        avg_loss = torch.stack([x["val_loss"] for x in outputs]).mean()
+        tensorboard_logs = {"val_loss": avg_loss}
+        return {"val_loss": avg_loss, "log": tensorboard_logs}
+
+    def test_step(self, batch, batch_idx):
+        image, tabular, y = batch
+
+        criterion = torch.nn.L1Loss()
+        y_pred = torch.flatten(self(image, tabular))
+        y_pred = y_pred.double()
+
+        test_loss = criterion(y_pred, y)
+
+        return {"test_loss": test_loss}
+
+    def test_epoch_end(self, outputs):
+        avg_loss = torch.stack([x["test_loss"] for x in outputs]).mean()
+        logs = {"test_loss": avg_loss}
+        return {"test_loss": avg_loss, "log": logs, "progress_bar": logs}
+
+    def setup(self, stage):
+
+        image_data = ImageDataset(pickle_file=f"{data_path}df.pkl", image_dir=f"{data_path}processed_images/")
+
+        train_size = int(0.80 * len(image_data))
+        val_size = int((len(image_data) - train_size) / 2)
+        test_size = int((len(image_data) - train_size) / 2)
+
+        self.train_set, self.val_set, self.test_set = random_split(image_data, (train_size, val_size, test_size))
+
     def configure_optimizers(self):
         return torch.optim.Adam(self.parameters(), lr=(self.lr))
 
+    def train_dataloader(self):
+        return DataLoader(self.train_set, batch_size=self.batch_size)
+
+    def val_dataloader(self):
+        return DataLoader(self.val_set, batch_size=self.batch_size)
+
+    def test_dataloader(self):
+        return DataLoader(self.test_set, batch_size=self.batch_size)
+
 
 if __name__ == "__main__":
-    image_data = ImageDataset(pickle_file=f"{data_path}df.pkl", image_dir=f"{data_path}processed_images/")
+    logger = TensorBoardLogger("lightning_logs", name="multi_input")
+    early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=5000, patience=7, verbose=False, mode="min")
 
     model = LitClassifier()
-    # mlflow_logger = pl_loggers.MLFlowLogger("logs/")
-    trainer = pl.Trainer(gpus=1)
+    trainer = pl.Trainer(gpus=1, logger=logger, early_stop_callback=early_stop_callback)
 
     lr_finder = trainer.lr_find(model)
-    print(lr_finder.results)
     fig = lr_finder.plot(suggest=True, show=True)
-
     new_lr = lr_finder.suggestion()
     print(new_lr)
-    model.hparams.lr = new_lr  # 1e-2
+    model.hparams.lr = new_lr
 
     trainer.fit(model)
+    trainer.test(model)