Skip to content

Commit

Permalink
fix PhysNet_preprocess_Video in image_preprocess
Browse files Browse the repository at this point in the history 
add Physnet based maml
  • Loading branch information
yoojin committed Dec 23, 2021
1 parent e37abae commit 6caab48
Show file tree
Hide file tree
Showing 7 changed files with 242 additions and 136 deletions.
108 changes: 43 additions & 65 deletions dataset/MetaPhysDataset.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,13 @@
import torchvision.transforms as transforms
from torchmeta.transforms import ClassSplitter

import scipy.io
from scipy.signal import butter

class MetaPhysDataset(MetaDataset):
def __init__(self, num_shots_tr, num_shots_ts, option='train',
unsupervised=0,frame_depth=10,
appearance_data=None, motion_data=None, target=None):
def __init__(self, num_shots_tr, num_shots_ts, video_data, label_data,
option='train', unsupervised=0,frame_depth=10,
):

self.transform = transforms.Compose([transforms.ToTensor()])
self.num_samples_per_task = num_shots_tr + num_shots_ts
Expand All @@ -18,70 +21,30 @@ def __init__(self, num_shots_tr, num_shots_ts, option='train',
self.num_shots_tr = num_shots_tr
self.num_shots_ts = num_shots_ts
self.unsupervised = unsupervised
self.a = appearance_data
self.m = motion_data
self.label = target
self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=num_shots_tr,
num_test_per_class=num_shots_ts)

def __getitem__(self, index):
if torch.is_tensor(index):
index = index.tolist()
ap = []
mo = []
la = []
data_len = len(self.label[index]) // self.num_samples_per_task # 1개의 데이터를 8개로
for i in range(self.num_samples_per_task):
ap.append(self.a[index][data_len * i:data_len * (i + 1)])
mo.append(self.m[index][data_len * i:data_len * (i + 1)])
la.append(self.label[index][data_len * i:data_len * (i + 1)])
self.video_data = video_data
self.label = label_data

task = PersonTask(ap, mo, la)

self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=self.num_shots_tr,
num_test_per_class=self.num_shots_ts)
if self.dataset_transform is not None:
task = self.dataset_transform(task)

return task

def __len__(self):
return len(self.label)

class MetaPhys_task_Dataset(MetaDataset):
def __init__(self, num_shots_tr, num_shots_ts, option='train',
unsupervised=0,frame_depth=10,
appearance_data=None, motion_data=None, target=None):

self.transform = transforms.Compose([transforms.ToTensor()])
self.num_samples_per_task = num_shots_tr + num_shots_ts
self.frame_depth = frame_depth
self.option = option
self.num_shots_tr = num_shots_tr
self.num_shots_ts = num_shots_ts
self.unsupervised = unsupervised
self.a = appearance_data
self.m = motion_data
self.label = target
self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=num_shots_tr,
num_test_per_class=num_shots_ts)

def __getitem__(self, index):
if torch.is_tensor(index):
index = index.tolist()
ap = []
mo = []

vi = []
la = []

for i in range(4):
ap.append(self.a[index][i])
mo.append(self.m[index][i])
la.append(self.label[index][i])
data_len = len(self.label[index]) // 2 # 1개의 데이터를 self.num_samples_per_task 로
for i in range(2):
vi.append(self.video_data[index][data_len * i:data_len * (i + 1)])
la.append(self.label[index][data_len * i:data_len * (i + 1)])

task = PersonTask(ap, mo, la)

self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=self.num_shots_tr,
num_test_per_class=self.num_shots_ts)

task = PersonTask(vi, la, len(vi))
if self.dataset_transform is not None:
task = self.dataset_transform(task)

Expand All @@ -90,28 +53,43 @@ def __getitem__(self, index):
def __len__(self):
return len(self.label)


class PersonTask(Task):
def __init__(self, a, m ,label):
def __init__(self, video ,label, num_samples):
super(PersonTask, self).__init__(None, None) # Regression task
self.transform = transforms.Compose([transforms.ToTensor()])
self.a = a
self.m = m
self.video = video
self.label = label
self.len_data = 0
self.num_samples = num_samples

def __len__(self):
return len(self.label)

def __getitem__(self, index):
self.len_data = len(self.label[index]) // 10
appearance_data = torch.tensor(np.transpose(self.a[index], (0, 3, 2, 1)), dtype=torch.float32)[:self.len_data*10]
motion_data = torch.tensor(np.transpose(self.m[index], (0, 3, 2, 1)), dtype=torch.float32)[:self.len_data*10]

target = torch.tensor(self.label[index], dtype=torch.float32)[:self.len_data*10]
input = torch.cat([appearance_data, motion_data], dim=1)
'''
if index<self.num_samples:
video_data = np.concatenate((self.video[index], self.video[index + 1]), axis=0)
label_data = np.concatenate((self.label[index], self.label[index + 1]), axis=0)
video_data = torch.tensor(np.transpose(video_data, (0, 4, 1, 2, 3)), dtype=torch.float32)
label_data = torch.tensor(label_data, dtype=torch.float32)
if torch.cuda.is_available():
video_data = video_data.to('cuda:9')
label_data = label_data.to('cuda:9')
return video_data, label_data
'''
video_data = torch.tensor(np.transpose(self.video[index], (0, 4, 1, 2, 3)), dtype=torch.float32)
#video_data = torch.tensor(np.transpose(self.video[index], (3, 0, 1, 2)), dtype=torch.float32)
label_data = torch.tensor(self.label[index], dtype=torch.float32)

if torch.cuda.is_available():
input = input.to('cuda:9')
target = target.to('cuda:9')
video_data = video_data.to('cuda:9')
label_data = label_data.to('cuda:9')

return video_data, label_data

return input, target
44 changes: 18 additions & 26 deletions dataset/dataset_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
from dataset.DeepPhysDataset import DeepPhysDataset
from dataset.PPNetDataset import PPNetDataset
from dataset.PhysNetDataset import PhysNetDataset
from dataset.MetaPhysDataset import MetaPhysDataset, MetaPhys_task_Dataset
from dataset.MetaPhysDataset import MetaPhysDataset
import random

def dataset_loader(save_root_path: str = "/media/hdd1/dy_dataset/",
model_name: str = "DeepPhys",
Expand Down Expand Up @@ -103,33 +104,24 @@ def dataset_loader(save_root_path: str = "/media/hdd1/dy_dataset/",
target=np.asarray(target_data)
)

elif model_name == "MetaPhys_task":
appearance_data_all = []
motion_data_all = []
target_data_all = []

for key in hpy_file.keys(): #1, 2, ...
appearance_data = []
motion_data = []
target_data = []
for data in hpy_file[key]: #1/1, 1/2, ...
appearance_data.append(hpy_file[key][data]['preprocessed_video'][:, :, :, -3:])
motion_data.append(hpy_file[key][data]['preprocessed_video'][:, :, :, :3])
target_data.append(hpy_file[key][data]['preprocessed_label'][:])
appearance_data_all.append(appearance_data) #np.asarray(
motion_data_all.append(motion_data)
target_data_all.append(target_data)
hpy_file.close()
elif model_name == "MetaPhysNet":
hpy_file = h5py.File(save_root_path + model_name + "_" + dataset_name + "_" + option + ".hdf5", "r")
video_data = []
label_data = []

dataset = MetaPhys_task_Dataset(num_shots,
keys = list(hpy_file.keys())
random_keys = random.sample(keys, len(keys))
for key in random_keys: #subject1, subject10, ...
video_data.append(hpy_file[key]['preprocessed_video'])
label_data.append(hpy_file[key]['preprocessed_label'])

dataset = MetaPhysDataset(num_shots,
num_test_shots,
option,
unsupervised,
frame_depth=10,
video_data=np.asarray(video_data[:]), # 너무 많을 때 개수 제한
label_data=np.asarray(label_data[:]),

appearance_data=np.asarray(appearance_data_all),
motion_data=np.asarray(motion_data_all),
target=np.asarray(target_data_all)
option=option,
unsupervised= unsupervised,
frame_depth=10
)

return dataset
44 changes: 18 additions & 26 deletions main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@

import numpy as np
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, random_split
from torchmeta.utils.data import BatchMetaDataLoader
from tqdm import tqdm
Expand All @@ -19,7 +20,7 @@
from utils.dataset_preprocess import preprocessing
from utils.funcs import normalize, plot_graph, detrend

from nets.models.MetaPhys import maml_train, maml_val, maml_test
from nets.models.Meta import Meta

with open('meta_params.json') as f:
jsonObject = json.load(f)
Expand Down Expand Up @@ -95,13 +96,11 @@
'''
if __TIME__:
start_time = time.time()
if model_params["name"] == 'MetaPhys':
if model_params["name"] == 'MetaPhys' or 'MetaPhysNet':
train_loader = BatchMetaDataLoader(train_dataset, batch_size=params["train_batch_size"],
shuffle=params["train_shuffle"])
validation_loader = BatchMetaDataLoader(validation_dataset, batch_size=params["train_batch_size"],
shuffle=params["train_shuffle"])
test_loader = BatchMetaDataLoader(test_dataset, batch_size=params["test_batch_size"],
shuffle=params["test_shuffle"])
else:
train_loader = DataLoader(train_dataset, batch_size=params["train_batch_size"],
shuffle=params["train_shuffle"])
Expand Down Expand Up @@ -132,8 +131,8 @@
# model = DataParallelModel(model, device_ids=[0, 1, 2])
# else:
# model = DataParallel(model, output_device=0)
torch.cuda.set_device(int(options["set_gpu_device"]))
model.cuda()
device = torch.device('cuda:9')
model.to(device=device)
else:
model = model.to('cpu')

Expand Down Expand Up @@ -165,7 +164,6 @@
if __TIME__:
start_time = time.time()
optimizer = optimizers(model.parameters(), hyper_params["learning_rate"], hyper_params["optimizer"])
inner_optimizer = optimizers(model.parameters(), hyper_params["inner_learning_rate"], hyper_params["inner_optimizer"])
scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.99)
if __TIME__:
log_info_time("setting optimizer time \t: ", datetime.timedelta(seconds=time.time() - start_time))
Expand All @@ -179,10 +177,10 @@
for epoch in range(hyper_params["epochs"]):
if __TIME__ and epoch == 0:
start_time = time.time()
with tqdm(train_loader, desc="Train ", total=len(train_loader)) as tepoch:
if model_params["name"] == 'MetaPhys':
maml_train(tepoch, model, inner_criterion, outer_criterion, inner_optimizer, optimizer, meta_params["num_adapt_steps"])
else:
if model_params["name"] == 'MetaPhys' or 'MetaPhysNet':
Meta(model, train_loader, validation_loader, inner_criterion)
else:
with tqdm(train_loader, desc="Train ", total=len(train_loader)) as tepoch:
model.train()
running_loss = 0.0
i = 0
Expand All @@ -206,13 +204,10 @@
running_loss += loss.item()
optimizer.step()
tepoch.set_postfix(loss=running_loss / params["train_batch_size"])
if __TIME__ and epoch == 0:
log_info_time("1 epoch training time \t: ", datetime.timedelta(seconds=time.time() - start_time))
if __TIME__ and epoch == 0:
log_info_time("1 epoch training time \t: ", datetime.timedelta(seconds=time.time() - start_time))

with tqdm(validation_loader, desc="Validation ", total=len(validation_loader)) as tepoch:
if model_params["name"] == 'MetaPhys':
maml_val(tepoch, model, inner_criterion, outer_criterion, inner_optimizer, meta_params["num_adapt_steps"])
else:
with tqdm(validation_loader, desc="Validation ", total=len(validation_loader)) as tepoch:
model.eval()
running_loss = 0.0
with torch.no_grad():
Expand Down Expand Up @@ -241,15 +236,12 @@
+ str(min_val_loss) + '.pth')
min_val_loss_model = copy.deepcopy(model)

if epoch + 1 == hyper_params["epochs"] or epoch % 10 == 0:
if __TIME__ and epoch == 0:
start_time = time.time()
if epoch + 1 == hyper_params["epochs"]:
model = min_val_loss_model
with tqdm(test_loader, desc="test ", total=len(test_loader)) as tepoch:
if model_params["name"] == 'MetaPhys':
maml_test(tepoch, model, inner_criterion, outer_criterion, inner_optimizer, meta_params["num_adapt_steps"])
else:
if epoch + 1 == hyper_params["epochs"] or epoch % 10 == 0:
if __TIME__ and epoch == 0:
start_time = time.time()
if epoch + 1 == hyper_params["epochs"]:
model = min_val_loss_model
with tqdm(test_loader, desc="test ", total=len(test_loader)) as tepoch:
model.eval()
inference_array = []
target_array = []
Expand Down
27 changes: 14 additions & 13 deletions meta_params.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,21 @@
"__MODEL_SUMMARY__" : 0,
"options":{
"parallel_criterion" : 1,
"parallel_criterion_comment" : "TODO need to verification"
"parallel_criterion_comment" : "TODO need to verification",
"set_gpu_device" : "9"
},
"params":
{
"save_root_path": "/media/hdd1/yj/dataset/",
"save_root_path": "/media/hdd1/yj/dataset2/",
"data_root_path": "/media/hdd1/",
"dataset_name": "cohface",
"dataset_name": "LGGI",
"checkpoint_path" : "/media/hdd1/yj/checkpoint/",
"train_ratio": 0.9,
"train_ratio_comment" : "generate train dataset using train_ratio",
"validation_ratio": 0.5,
"validation_ratio": 0.6,
"validation_ratio_comment" : "split train dataset using validation_ratio",
"train_batch_size" : 1,
"train_shuffle" : 1,
"train_shuffle" : 0,
"test_batch_size" : 1,
"test_shuffle" : 0
},
Expand All @@ -39,8 +40,8 @@
"adam","sgd","rms_prop","ada_delta","ada_grad","ada_max",
"ada_mw","a_sgd","lbfgs","n_adam","r_adam","rprop","sparse_adam"
],
"learning_rate": 0.001,
"inner_learning_rate" : 0.01,
"learning_rate": 0.003,
"inner_learning_rate" : 0.05,
"learning_rate_comment": [
"DeepPhys : lr = 1",
"PhysNet : lr = 0.001",
Expand All @@ -50,26 +51,26 @@
},
"meta_params":
{
"num_shots" : 2,
"num_test_shots" : 2,
"num_shots" : 1,
"num_test_shots" : 1,
"fs" : 30,
"fs_comment" : "sampling rate of dataset",
"unsupervised" : 0,
"pre_trained" : 0,
"pre_trained_comment" : "pretrained or not",
"inner_loss" : "mse",
"outer_loss" : "mse",
"inner_loss" : "neg_pearson",
"outer_loss" : "neg_pearson",
"num_adapt_steps" : 1
},
"model_params":
{
"name": "MetaPhys_task",
"name": "MetaPhysNet",
"name_comment":
[
"DeepPhys",
"PhysNet",
"MetaPhys",
"MetaPhys_task"
"MetaPhysNet"
]
}
}
Loading

0 comments on commit 6caab48

Please sign in to comment.