add TODO list...

dataset/DeepPhysDataset.py

@@ -19,10 +19,10 @@ def __getitem__(self, index):
         motion_data = torch.tensor(np.transpose(self.m[index], (2, 0, 1)), dtype=torch.float32)
         target = torch.tensor(self.label[index], dtype=torch.float32)
 
-        inputs = [appearance_data, motion_data]
+        inputs = torch.stack([appearance_data,motion_data],dim=0)
 
         if torch.cuda.is_available():
-            inputs.to('cuda')
+            inputs = inputs.to('cuda')
             target = target.to('cuda')
 
         return inputs, target

log.py

@@ -7,3 +7,7 @@ def log_info_time(message, time):
 
 def log_warning(message):
     print(Fore.LIGHTRED_EX + Style.BRIGHT + message + Style.RESET_ALL)
+
+
+def log_info(message):
+    print(Fore.LIGHTYELLOW_EX + Style.BRIGHT + message + Style.RESET_ALL)

main.py

@@ -3,6 +3,7 @@
 import time
 
 import numpy as np
+import scipy.signal
 import torch
 from torch.utils.data import DataLoader, random_split
 from tqdm import tqdm
@@ -13,7 +14,7 @@
 from models import is_model_support, get_model, summary
 from optim import optimizer
 from utils.dataset_preprocess import preprocessing
-from utils.funcs import normalize, plot_graph
+from utils.funcs import normalize, plot_graph, detrend
 
 with open('params.json') as f:
     jsonObject = json.load(f)
@@ -67,6 +68,7 @@
 if __TIME__:
     start_time = time.time()
 test_dataset = dataset_loader(save_root_path=params["save_root_path"],
+                              model_name=model_params["name"],
                               dataset_name=params["dataset_name"],
                               option="test")
 if __TIME__:
@@ -104,7 +106,7 @@
     model = model.to('cpu')
 
 if __MODEL_SUMMARY__:
-    summary(model)
+    summary(model,model_params["name"])
 
 if __TIME__:
     log_info_time("model initialize time \t: ", datetime.timedelta(seconds=time.time() - start_time))
@@ -142,7 +144,7 @@
     if __TIME__ and epoch == 0:
         start_time = time.time()
     with tqdm(train_loader, desc="Train ", total=len(train_loader)) as tepoch:
-        model.train()
+        # model.train()
         running_loss = 0.0
         for inputs, target in tepoch:
             tepoch.set_description(f"Train Epoch {epoch}")
@@ -158,7 +160,7 @@
         log_info_time("1 epoch training time \t: ", datetime.timedelta(seconds=time.time() - start_time))
 
     with tqdm(validation_loader, desc="Validation ", total=len(validation_loader)) as tepoch:
-        model.eval()
+        # model.eval()
         running_loss = 0.0
         with torch.no_grad():
             for inputs, target in tepoch:
@@ -180,7 +182,7 @@
         if __TIME__ and epoch == 0:
             start_time = time.time()
         with tqdm(test_loader, desc="test ", total=len(test_loader)) as tepoch:
-            model.eval()
+            # model.eval()
             inference_array = []
             target_array = []
             with torch.no_grad():
@@ -191,11 +193,15 @@
                     running_loss += loss.item()
                     tepoch.set_postfix(loss=running_loss / (params["train_batch_size"] / params["test_batch_size"]))
                     if model_params["name"] == "PhysNet":
-                        inference_array.extend(normalize(outputs.numpy()))
-                        target_array.extend(normalize(target.numpy()))
+                        inference_array.extend(normalize(outputs.cpu().numpy()))
+                        target_array.extend(normalize(target.cpu().numpy()))
                     else:
-                        inference_array.extend(outputs.numpy())
-                        target_array.extend(target.numpy())
-            plot_graph(0, len(inference_array), target_array, inference_array)
-        if __TIME__:
+                        inference_array.extend(outputs.cpu().numpy())
+                        target_array.extend(target.cpu().numpy())
+            if model_params["name"] == "DeepPhys":
+                inference_array = scipy.signal.detrend(np.cumsum(inference_array))
+                target_array = scipy.signal.detrend(np.cumsum(target_array))
+
+            plot_graph(0, 300, target_array, inference_array)
+        if __TIME__ and epoch == 0:
             log_info_time("inference time \t: ", datetime.timedelta(seconds=time.time() - start_time))

models.py

@@ -1,6 +1,6 @@
 import torchsummary
 
-from log import log_warning
+from log import log_warning, log_info
 from nets.models.DeepPhys import DeepPhys
 from nets.models.DeepPhys_DA import DeepPhys_DA
 from nets.models.PhysNet import PhysNet
@@ -39,6 +39,9 @@ def summary(model, model_name):
     :param model_name: implemented model name
     :return: model
     """
+    log_info("=========================================")
+    log_info(model_name)
+    log_info("=========================================")
     if model_name == "DeepPhys" or model_name == DeepPhys_DA:
         torchsummary.summary(model, (2, 3, 36, 36))
     elif model_name == "PhsNet":

nets/models/DeepPhys.py

@@ -29,8 +29,9 @@ def forward(self, inputs):
         :return:
         original 2d model
         """
-        self.attention_mask1, self.attention_mask2 = self.appearance_model(inputs[0])
-        motion_output = self.motion_model(inputs[1], self.attention_mask1, self.attention_mask2)
+        inputs = torch.chunk(inputs,2,dim=1)
+        self.attention_mask1, self.attention_mask2 = self.appearance_model(torch.squeeze(inputs[0],1))
+        motion_output = self.motion_model(torch.squeeze(inputs[1],1), self.attention_mask1, self.attention_mask2)
         out = self.linear_model(motion_output)
 
         return out

utils/image_preprocess.py

@@ -117,7 +117,9 @@ def generate_MotionDifference(prev_frame, crop_frame):
     '''
     # motion input
     motion_input = (crop_frame - prev_frame) / (crop_frame + prev_frame)
+    # TODO : need to diminish outliers [ clipping ]
     motion_input = motion_input / np.std(motion_input)
+    # TODO : do not divide each D frame, modify divide whole video's unit standard deviataion
     return motion_input
 
 

utils/text_preprocess.py

@@ -6,6 +6,8 @@ def Deepphys_preprocess_Label(path):
     :param path: label file path
     :return: delta pulse
     '''
+    # TODO : need to check length with video frames
+    # TODO : need to implement piecewise cubic Hermite interpolation
     # Load input
     f = open(path, 'r')
     f_read = f.read().split('\n')
@@ -21,17 +23,6 @@ def Deepphys_preprocess_Label(path):
     delta_pulse = delta_label.copy()  # 이거 왜 있지?
     f.close()
 
-    # Normalize
-    # part = 0
-    # window = 32
-    # while part < (len(delta_pulse) // window) - 1:
-    #     delta_pulse[part * window:(part + 1) * window] -= np.mean(delta_pulse[part * window:(part + 1) * window])
-    #     delta_pulse[part * window:(part + 1) * window] /= np.std(delta_pulse[part * window:(part + 1) * window])
-    #     part += 1
-    # if len(delta_pulse) % window != 0:
-    #     delta_pulse[part * window:] -= np.mean(delta_pulse[part * window:])
-    #     delta_pulse[part * window:] /= np.std(delta_pulse[part * window:])
-
     return delta_pulse