From eaa066fcb02f46c4f09013161116f3d34adac8e8 Mon Sep 17 00:00:00 2001 From: yoojin Date: Fri, 13 Aug 2021 16:08:44 +0900 Subject: [PATCH] modify MetaPhys --- Meta_main.py | 248 ----------------------- dataset/MetaPhysDataset.py | 147 ++++---------- dataset/dataset_loader.py | 41 ++-- main.py | 115 +++++++---- meta_params.json | 4 +- nets/models/MetaPhys.py | 71 +++++++ utils/Meta_class_splitters.py | 360 ---------------------------------- utils/dataset_preprocess.py | 55 ------ 8 files changed, 207 insertions(+), 834 deletions(-) delete mode 100644 Meta_main.py delete mode 100644 utils/Meta_class_splitters.py diff --git a/Meta_main.py b/Meta_main.py deleted file mode 100644 index 8d6dc05..0000000 --- a/Meta_main.py +++ /dev/null @@ -1,248 +0,0 @@ -import datetime -import json -import time - -import numpy as np -import torch -from torch.utils.data import DataLoader, random_split -from torchmeta.utils.data import BatchMetaDataLoader -from tqdm import tqdm -import higher - -from dataset.dataset_loader import dataset_loader -from log import log_info_time -from loss import loss_fn -from models import is_model_support, get_model, summary -from optim import optimizers -from torch.optim import lr_scheduler -from utils.dataset_preprocess import preprocessing,Meta_preprocessing -from utils.funcs import normalize, plot_graph, detrend - -with open('meta_params.json') as f: - jsonObject = json.load(f) - __PREPROCESSING__ = jsonObject.get("__PREPROCESSING__") - __TIME__ = jsonObject.get("__TIME__") - __MODEL_SUMMARY__ = jsonObject.get("__MODEL_SUMMARY__") - options = jsonObject.get("options") - params = jsonObject.get("params") - hyper_params = jsonObject.get("hyper_params") - model_params = jsonObject.get("model_params") - meta_params = jsonObject.get("meta_params") -# -""" -Check Model Support -""" -is_model_support(model_params["name"], model_params["name_comment"]) -''' -Generate preprocessed data hpy file -''' -if __PREPROCESSING__: - if __TIME__: - start_time = time.time() - - Meta_preprocessing(save_root_path=params["save_root_path"], - model_name=model_params["name"], - data_root_path=params["data_root_path"], - dataset_name=params["dataset_name"]) - - if __TIME__: - log_info_time("preprocessing time \t:", datetime.timedelta(seconds=time.time() - start_time)) - -''' -Load dataset before using Torch DataLoader -''' -if __TIME__: - start_time = time.time() - -dataset = dataset_loader(save_root_path=params["save_root_path"], - model_name=model_params["name"], - dataset_name=params["dataset_name"], - option="train", - - num_shots= meta_params["num_shots"], - num_test_shots = meta_params["num_test_shots"], - fs = meta_params["fs"], - unsupervised = meta_params["unsupervised"] - ) - -train_dataset, validation_dataset = random_split(dataset, - [int(np.floor( - len(dataset) * params["validation_ratio"])), - int(np.ceil( - len(dataset) * (1 - params["validation_ratio"])))] - ) -if __TIME__: - log_info_time("load train hpy time \t: ", datetime.timedelta(seconds=time.time() - start_time)) - -if __TIME__: - start_time = time.time() -test_dataset = dataset_loader(save_root_path=params["save_root_path"], - model_name=model_params["name"], - dataset_name=params["dataset_name"], - option="test", - - num_shots=meta_params["num_shots"], - num_test_shots=meta_params["num_test_shots"], - fs=meta_params["fs"], - unsupervised=meta_params["unsupervised"] - ) -if __TIME__: - log_info_time("load test hpy time \t: ", datetime.timedelta(seconds=time.time() - start_time)) - -''' - Call dataloader for iterate dataset -''' -if __TIME__: - start_time = time.time() - -train_loader = BatchMetaDataLoader(train_dataset, batch_size=params["train_batch_size"], - shuffle=params["train_shuffle"]) -validation_loader = BatchMetaDataLoader(validation_dataset, batch_size=params["train_batch_size"], - shuffle=params["train_shuffle"]) -test_loader = BatchMetaDataLoader(test_dataset, batch_size=params["test_batch_size"], - shuffle=params["test_shuffle"]) - -if __TIME__: - log_info_time("generate dataloader time \t: ", datetime.timedelta(seconds=time.time() - start_time)) -''' -Setting Learning Model -''' -if __TIME__: - start_time = time.time() -model = get_model(model_params["name"]) - -if meta_params["pre_trained"] ==1: - print('Using pre-trained on all ALL AFRL!') - model.load_state_dict(torch.load('./checkpoints/AFRL_pretrained/meta_pretrained_all_AFRL.pth')) -else: - print('Not using any pretrained models') - -if torch.cuda.is_available(): - # os.environ["CUDA_VISIBLE_DEVICES"] = '0, 1, 2, 3, 4, 5, 6, 7, 8, 9' - # TODO: implement parallel training - # if options["parallel_criterion"] : - # print(options["parallel_criterion_comment"]) - # model = DataParallelModel(model, device_ids=[0, 1, 2]) - # else: - # model = DataParallel(model, output_device=0) - device = torch.device('cuda:9') - model.to(device) -else: - model = model.to('cpu') - -if __MODEL_SUMMARY__: - summary(model,model_params["name"]) - -if __TIME__: - log_info_time("model initialize time \t: ", datetime.timedelta(seconds=time.time() - start_time)) - -''' -Setting Loss Function -''' -if __TIME__: - start_time = time.time() -criterion = loss_fn(hyper_params["loss_fn"]) -inner_criterion = loss_fn(meta_params["inner_loss"]) -outer_criterion = loss_fn(meta_params["outer_loss"]) - -if __TIME__: - log_info_time("setting loss func time \t: ", datetime.timedelta(seconds=time.time() - start_time)) -''' -Setting Optimizer -''' -if __TIME__: - start_time = time.time() -optimizer = optimizers(model.parameters(), hyper_params["learning_rate"], hyper_params["optimizer"]) -inner_optimizer = optimizers(model.parameters(), hyper_params["inner_learning_rate"], hyper_params["inner_optimizer"]) -scheduler = lr_scheduler.ExponentialLR(optimizer,gamma=0.99) -if __TIME__: - log_info_time("setting optimizer time \t: ", datetime.timedelta(seconds=time.time() - start_time)) - -''' -Model Training Step -''' - -min_val_loss = 100.0 - -for epoch in range(hyper_params["epochs"]): - if __TIME__ and epoch == 0: - start_time = time.time() - with tqdm(train_loader, desc="Train ", total=len(train_loader)) as tepoch: - model.train() - for batch_idx, batch in enumerate(tepoch): - tepoch.set_description(f"Train Epoch {epoch}") - outer_loss = 0.0 - - batch['train'][0] = batch['train'][0].view(params["train_batch_size"], -1, 6, 36, 36) - batch['test'][0] = batch['test'][0].view(params["train_batch_size"], -1, 6, 36, 36) - batch['train'][1] = batch['train'][1].view(params["train_batch_size"], -1, 1) - batch['test'][1] = batch['test'][1].view(params["train_batch_size"], -1, 1) - - inputs, targets = batch['train'] - inputs = inputs.to(device=device) - targets = targets.to(device=device) - - test_inputs, test_targets= batch['test'] - test_inputs = test_inputs.to(device=device) - test_targets = test_targets.to(device=device) - - for task_idx, (input, target, test_input, test_target) in enumerate(zip(inputs, targets, test_inputs, test_targets)): - with higher.innerloop_ctx(model, inner_optimizer, copy_initial_weights=False) as (fmodel, diffopt): - for step in range(meta_params["num_adapt_steps"]): - train_logit = fmodel(input) - inner_loss = inner_criterion(train_logit, target) - diffopt.step(inner_loss) - test_logit = fmodel(test_input) - outer_loss += outer_criterion(test_logit, test_target) - - optimizer.zero_grad() - maml_loss = outer_loss / len(validation_loader) - maml_loss.backward() - optimizer.step() - tepoch.set_postfix(loss=outer_loss / params["train_batch_size"]) - if __TIME__ and epoch == 0: - log_info_time("1 epoch training time \t: ", datetime.timedelta(seconds=time.time() - start_time)) - - with tqdm(validation_loader, desc="Validation ", total=len(validation_loader)) as tepoch: - model.train() - #running_loss = 0.0 - #with torch.no_grad(): - for batch_idx, batch in enumerate(tepoch): - tepoch.set_description(f"Validation") - outer_loss = 0.0 - - batch['train'][0] = batch['train'][0].view(params["train_batch_size"], -1, 6, 36, 36) - batch['test'][0] = batch['test'][0].view(params["train_batch_size"], -1, 6, 36, 36) - batch['train'][1] = batch['train'][1].view(params["train_batch_size"], -1, 1) - batch['test'][1] = batch['test'][1].view(params["train_batch_size"], -1, 1) - - inputs, targets = batch['train'] - - inputs = inputs.to(device=device) - targets = targets.to(device=device) - - test_inputs, test_targets = batch['test'] - - for task_idx, (input, target, test_input, test_target) in enumerate(zip(inputs, targets, test_inputs, test_targets)): - with higher.innerloop_ctx(model, inner_optimizer, copy_initial_weights=False) as (fmodel, diffopt): - for step in range(meta_params["num_adapt_steps"]): - train_logit = fmodel(input) - inner_loss = inner_criterion(train_logit, target) - diffopt.step(inner_loss) - test_data_loader = DataLoader(test_input, batch_size=params["test_batch_size"], shuffle=False) - test_logits = torch.tensor([], device=device) - for i, test_batch in enumerate(test_data_loader): - pred = fmodel(test_batch.to(device=device)).detach() - test_logits = torch.cat((test_logits, pred), 0) - temp_test_loss = outer_criterion(test_logits, test_target.to(device=device)) - outer_loss += temp_test_loss - tepoch.set_postfix(loss=outer_loss / len(validation_loader)) - - if min_val_loss > outer_loss: # save the train model - min_val_loss = outer_loss - checkpoint = {'Epoch': epoch, - 'state_dict': model.state_dict(), - 'optimizer': optimizer.state_dict()} - torch.save(checkpoint, params["checkpoint_path"] + model_params["name"] + "/" - + params["dataset_name"] + "_" + str(epoch) + "_" - + str(min_val_loss) + '.pth') diff --git a/dataset/MetaPhysDataset.py b/dataset/MetaPhysDataset.py index bd0910f..054af28 100644 --- a/dataset/MetaPhysDataset.py +++ b/dataset/MetaPhysDataset.py @@ -1,136 +1,75 @@ -import random - -import h5py +import torch import numpy as np -import scipy.io -from scipy.signal import butter from torchmeta.utils.data import Task, MetaDataset import torchvision.transforms as transforms - -from utils.Meta_class_splitters import ClassSplitter -from utils.funcs import ToTensor1D - -np.random.seed(100) +from torchmeta.transforms import ClassSplitter class MetaPhysDataset(MetaDataset): - """ - Simple regression task, based on sinusoids, as introduced in [1]. - - Parameters - ---------- - num_samples_per_task : int - Number of examples per task. - - num_tasks : int (default: 1,000,000) - Overall number of tasks to sample. - - transform : callable, optional - A function/transform that takes a numpy array of size (1,) and returns a - transformed version of the input. - - target_transform : callable, optional - A function/transform that takes a numpy array of size (1,) and returns a - transformed version of the target. + def __init__(self, num_shots_tr, num_shots_ts, option='train', + fs=30, unsupervised=0,frame_depth=10, + appearance_data=None, motion_data=None, target=None): - dataset_transform : callable, optional - A function/transform that takes a dataset (ie. a task), and returns a - transformed version of it. E.g. `torchmeta.transforms.ClassSplitter()`. - """ - - def __init__(self, num_shots_tr, num_shots_ts, person_data_path, option='train', - fs=30, unsupervised=0,batch_size = 1,frame_depth=10,random_seed =10): - super(MetaPhysDataset, self).__init__(meta_split='train', target_transform=ToTensor1D()) - self.transform = ToTensor1D() + self.transform = transforms.Compose([transforms.ToTensor()]) self.num_samples_per_task = num_shots_tr + num_shots_ts - self.person_data_path = person_data_path self.frame_depth = frame_depth self.fs = fs self.option = option self.num_shots_tr = num_shots_tr + self.num_shots_ts = num_shots_ts self.unsupervised = unsupervised - np.random.seed(random_seed) - if self.option == 'train': - self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=num_shots_tr, + self.a = appearance_data + self.m = motion_data + self.label = target + self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=num_shots_tr, num_test_per_class=num_shots_ts) - def __len__(self): - return len(self.person_data_path) - - def __getitem__(self, index): - per_task_data = self.person_data_path[index] - - if self.option == 'test': - self.num_shots_ts = len(per_task_data) - self.num_shots_tr - self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=self.num_shots_tr, - num_test_per_class=self.num_shots_ts) - self.num_samples_per_task = self.num_shots_tr + self.num_shots_ts - - if self.option == 'train': - random.shuffle(per_task_data) - - self.num_shots_ts = len(per_task_data) - self.num_shots_tr - if self.num_shots_ts > 8: - self.num_shots_ts = 8 - self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=self.num_shots_tr, - num_test_per_class=self.num_shots_ts) - self.num_samples_per_task = self.num_shots_tr + self.num_shots_ts - - task_path = per_task_data[:self.num_samples_per_task] - task = PersonTask(self.num_samples_per_task, task_path, self.num_shots_tr, frame_depth=self.frame_depth, fs=self.fs, option=self.option, unsupervised=self.unsupervised) + if torch.is_tensor(index): + index = index.tolist() + + self.dataset_transform = ClassSplitter(shuffle=False, num_train_per_class=self.num_shots_tr, + num_test_per_class=self.num_shots_ts) + ap = [] + mo = [] + la = [] + data_len = len(self.label[index]) // self.num_samples_per_task # 1개의 데이터를 8개로 + for i in range(self.num_samples_per_task): + ap.append(self.a[index][data_len * i:data_len * (i + 1)]) + mo.append(self.m[index][data_len * i:data_len * (i + 1)]) + la.append(self.label[index][data_len * i:data_len * (i + 1)]) + + task = PersonTask(ap, mo, la) if self.dataset_transform is not None: task = self.dataset_transform(task) + return task + def __len__(self): + return len(self.label) class PersonTask(Task): - def __init__(self, num_samples, task_data_path, num_shots_tr, frame_depth=10, - fs=30, option='train', unsupervised=0): + def __init__(self, a, m ,label): super(PersonTask, self).__init__(None, None) # Regression task - self.num_shots_tr = num_shots_tr - self.num_samples = num_samples - self.transform = ToTensor1D() - self.target_transform = ToTensor1D() - self.task_data_path = task_data_path - self.frame_depth = frame_depth - self.fs = fs - self.option = option - self.unsupervised = unsupervised + self.a = a + self.m = m + self.label = label self.len_data = 0 def __len__(self): - return self.num_samples + return len(self.label) def __getitem__(self, index): - temp_path = self.task_data_path[index] - f1 = h5py.File(temp_path, 'r') - - self.len_data = len(f1["preprocessed_video"]) // 10 - output = np.transpose(np.array(f1["preprocessed_video"]), [0, 3, 2, 1])[:self.len_data*10] - label = np.array(f1["preprocessed_label"])[: self.len_data * 10] - - [b, a] = butter(1, [0.75 / self.fs * 2, 2.5 / self.fs * 2], btype='bandpass') - label = scipy.signal.filtfilt(b, a, np.squeeze(label)) - label = np.expand_dims(label, axis=1) - - # Average the frame - motion_data = output[:, :3, :, :] - apperance_data = output[:, 3:, :, :] - apperance_data = np.reshape(apperance_data, (self.len_data, self.frame_depth, 3, 36, 36)) - apperance_data = np.average(apperance_data, axis=1) - apperance_data = np.repeat(apperance_data[:, np.newaxis, :, :, :], self.frame_depth, axis=1) - apperance_data = np.reshape(apperance_data, (apperance_data.shape[0] * apperance_data.shape[1], - apperance_data.shape[2], apperance_data.shape[3], - apperance_data.shape[4])) - output = np.concatenate((motion_data, apperance_data), axis=1) - - if self.transform is not None: - output = self.transform(output) + self.len_data = len(self.label[index]) // 10 + appearance_data = torch.tensor(np.transpose(self.a[index], (0, 3, 2, 1)), dtype=torch.float32)[:self.len_data*10] + motion_data = torch.tensor(np.transpose(self.m[index], (0, 3, 2, 1)), dtype=torch.float32)[:self.len_data*10] - if self.target_transform is not None: - label = self.target_transform(label) + target = torch.tensor(self.label[index], dtype=torch.float32)[:self.len_data*10] + input = torch.cat([appearance_data, motion_data], dim=1) - return output, label + if torch.cuda.is_available(): + input = input.to('cuda:9') + target = target.to('cuda:9') + return input, target \ No newline at end of file diff --git a/dataset/dataset_loader.py b/dataset/dataset_loader.py index 6320bbc..1b1cdaf 100755 --- a/dataset/dataset_loader.py +++ b/dataset/dataset_loader.py @@ -12,10 +12,10 @@ def dataset_loader(save_root_path: str = "/media/hdd1/dy_dataset/", option: str = "train", num_shots: int = 6, - num_test_shots:int = 6, + num_test_shots:int =2, fs: int = 30, - unsupervised: int = 0, - batch_size = 1): + unsupervised: int = 0 + ): ''' :param save_root_path: save file destination path :param model_name : model_name @@ -23,9 +23,9 @@ def dataset_loader(save_root_path: str = "/media/hdd1/dy_dataset/", :param option:[train, test] :return: dataset ''' + hpy_file = h5py.File(save_root_path + model_name + "_" + dataset_name + "_" + option + ".hdf5", "r") if model_name == "DeepPhys": - hpy_file = h5py.File(save_root_path + model_name + "_" + dataset_name + "_" + option + ".hdf5", "r") appearance_data = [] motion_data = [] target_data = [] @@ -35,12 +35,10 @@ def dataset_loader(save_root_path: str = "/media/hdd1/dy_dataset/", motion_data.extend(hpy_file[key]['preprocessed_video'][:, :, :, :3]) target_data.extend(hpy_file[key]['preprocessed_label']) hpy_file.close() - dataset = DeepPhysDataset(appearance_data=np.asarray(appearance_data), motion_data=np.asarray(motion_data), target=np.asarray(target_data)) elif model_name == "PhysNet" or model_name == "PhysNet_LSTM": - hpy_file = h5py.File(save_root_path + model_name + "_" + dataset_name + "_" + option + ".hdf5", "r") video_data = [] label_data = [] for key in hpy_file.keys(): @@ -52,31 +50,26 @@ def dataset_loader(save_root_path: str = "/media/hdd1/dy_dataset/", label_data=np.asarray(label_data)) if model_name == "MetaPhys": - train_path = [] - test_path = [] + appearance_data = [] + motion_data = [] + target_data = [] - hpy_filelist = save_root_path + model_name + "/" + dataset_name+'/' - for i in sorted(os.listdir(hpy_filelist)[:35]): - path = [] - for j in sorted(os.listdir(hpy_filelist + i)): - path.append(hpy_filelist + i + '/' + j) - train_path.append(path) - for i in sorted(os.listdir(hpy_filelist)[35:]): - path = [] - for j in sorted(os.listdir(hpy_filelist + i)): - path.append(hpy_filelist + i + '/' + j) - test_path.append(path) + for key in hpy_file.keys(): #subject1, subject10, ... + appearance_data.append(hpy_file[key]['preprocessed_video'][:, :, :, -3:]) + motion_data.append(hpy_file[key]['preprocessed_video'][:, :, :, :3]) + target_data.append(hpy_file[key]['preprocessed_label'][:]) + hpy_file.close() dataset = MetaPhysDataset(num_shots, num_test_shots, - train_path, option, fs, unsupervised, - batch_size, - frame_depth=10 - #transform=transform, - # target_transform=transform, + frame_depth=10, + + appearance_data=np.asarray(appearance_data), + motion_data=np.asarray(motion_data), + target=np.asarray(target_data) ) return dataset diff --git a/main.py b/main.py index d8d700d..cec7e0f 100755 --- a/main.py +++ b/main.py @@ -5,18 +5,21 @@ import numpy as np import torch from torch.utils.data import DataLoader, random_split +from torchmeta.utils.data import BatchMetaDataLoader from tqdm import tqdm from dataset.dataset_loader import dataset_loader from log import log_info_time from loss import loss_fn from models import is_model_support, get_model, summary -from optim import optimizer +from optim import optimizers from torch.optim import lr_scheduler from utils.dataset_preprocess import preprocessing from utils.funcs import normalize, plot_graph, detrend -with open('params.json') as f: +from nets.models.MetaPhys import maml_train, maml_val + +with open('meta_params.json') as f: jsonObject = json.load(f) __PREPROCESSING__ = jsonObject.get("__PREPROCESSING__") __TIME__ = jsonObject.get("__TIME__") @@ -25,6 +28,7 @@ params = jsonObject.get("params") hyper_params = jsonObject.get("hyper_params") model_params = jsonObject.get("model_params") + meta_params = jsonObject.get("meta_params") # """ Check Model Support @@ -54,7 +58,13 @@ dataset = dataset_loader(save_root_path=params["save_root_path"], model_name=model_params["name"], dataset_name=params["dataset_name"], - option="train") + option="train", + + num_shots=meta_params["num_shots"], + num_test_shots=meta_params["num_test_shots"], + fs=meta_params["fs"], + unsupervised=meta_params["unsupervised"] + ) train_dataset, validation_dataset = random_split(dataset, [int(np.floor( @@ -79,12 +89,20 @@ ''' if __TIME__: start_time = time.time() -train_loader = DataLoader(train_dataset, batch_size=params["train_batch_size"], - shuffle=params["train_shuffle"]) -validation_loader = DataLoader(validation_dataset, batch_size=params["train_batch_size"], - shuffle=params["train_shuffle"]) -test_loader = DataLoader(test_dataset, batch_size=params["test_batch_size"], - shuffle=params["test_shuffle"]) +if model_params["name"] == 'MetaPhys': + train_loader = BatchMetaDataLoader(train_dataset, batch_size=params["train_batch_size"], + shuffle=params["train_shuffle"]) + validation_loader = BatchMetaDataLoader(validation_dataset, batch_size=params["train_batch_size"], + shuffle=params["train_shuffle"]) + test_loader = BatchMetaDataLoader(test_dataset, batch_size=params["test_batch_size"], + shuffle=params["test_shuffle"]) +else: + train_loader = DataLoader(train_dataset, batch_size=params["train_batch_size"], + shuffle=params["train_shuffle"]) + validation_loader = DataLoader(validation_dataset, batch_size=params["train_batch_size"], + shuffle=params["train_shuffle"]) + test_loader = DataLoader(test_dataset, batch_size=params["test_batch_size"], + shuffle=params["test_shuffle"]) if __TIME__: log_info_time("generate dataloader time \t: ", datetime.timedelta(seconds=time.time() - start_time)) ''' @@ -93,6 +111,13 @@ if __TIME__: start_time = time.time() model = get_model(model_params["name"]) + +if meta_params["pre_trained"] == 1: + print('Using pre-trained on all ALL AFRL!') + model.load_state_dict(torch.load('./checkpoints/AFRL_pretrained/meta_pretrained_all_AFRL.pth')) +else: + print('Not using any pretrained models') + if torch.cuda.is_available(): # os.environ["CUDA_VISIBLE_DEVICES"] = '0, 1, 2, 3, 4, 5, 6, 7, 8, 9' # TODO: implement parallel training @@ -101,7 +126,8 @@ # model = DataParallelModel(model, device_ids=[0, 1, 2]) # else: # model = DataParallel(model, output_device=0) - model.cuda() + device = torch.device('cuda:9') + model.to(device=device) else: model = model.to('cpu') @@ -117,7 +143,8 @@ if __TIME__: start_time = time.time() criterion = loss_fn(hyper_params["loss_fn"]) - +inner_criterion = loss_fn(meta_params["inner_loss"]) +outer_criterion = loss_fn(meta_params["outer_loss"]) # if torch.cuda.is_available(): # TODO: implement parallel training # if options["parallel_criterion"] : @@ -131,8 +158,9 @@ ''' if __TIME__: start_time = time.time() -optimizer = optimizer(model.parameters(), hyper_params["learning_rate"], hyper_params["optimizer"]) -scheduler = lr_scheduler.ExponentialLR(optimizer,gamma=0.99) +optimizer = optimizers(model.parameters(), hyper_params["learning_rate"], hyper_params["optimizer"]) +inner_optimizer = optimizers(model.parameters(), hyper_params["inner_learning_rate"], hyper_params["inner_optimizer"]) +scheduler = lr_scheduler.ExponentialLR(optimizer, gamma=0.99) if __TIME__: log_info_time("setting optimizer time \t: ", datetime.timedelta(seconds=time.time() - start_time)) @@ -145,39 +173,44 @@ if __TIME__ and epoch == 0: start_time = time.time() with tqdm(train_loader, desc="Train ", total=len(train_loader)) as tepoch: - model.train() - running_loss = 0.0 - for inputs, target in tepoch: - tepoch.set_description(f"Train Epoch {epoch}") - outputs = model(inputs) - loss = criterion(outputs, target) - - optimizer.zero_grad() - loss.backward() - running_loss += loss.item() - optimizer.step() - tepoch.set_postfix(loss=running_loss / params["train_batch_size"]) - if __TIME__ and epoch == 0: - log_info_time("1 epoch training time \t: ", datetime.timedelta(seconds=time.time() - start_time)) - - with tqdm(validation_loader, desc="Validation ", total=len(validation_loader)) as tepoch: - model.eval() - running_loss = 0.0 - with torch.no_grad(): + if model_params["name"] == 'MetaPhys': + maml_train(tepoch, model, inner_criterion, outer_criterion, inner_optimizer,optimizer, meta_params["num_adapt_steps"]) + else: + model.train() + running_loss = 0.0 for inputs, target in tepoch: - tepoch.set_description(f"Validation") + tepoch.set_description(f"Train Epoch {epoch}") outputs = model(inputs) loss = criterion(outputs, target) + optimizer.zero_grad() + loss.backward() running_loss += loss.item() + optimizer.step() tepoch.set_postfix(loss=running_loss / params["train_batch_size"]) - if min_val_loss > running_loss: # save the train model - min_val_loss = running_loss - checkpoint = {'Epoch': epoch, - 'state_dict': model.state_dict(), - 'optimizer': optimizer.state_dict()} - torch.save(checkpoint, params["checkpoint_path"] + model_params["name"] + "/" - + params["dataset_name"] + "_" + str(epoch) + "_" - + str(min_val_loss) + '.pth') + if __TIME__ and epoch == 0: + log_info_time("1 epoch training time \t: ", datetime.timedelta(seconds=time.time() - start_time)) + + with tqdm(validation_loader, desc="Validation ", total=len(validation_loader)) as tepoch: + if model_params["name"] == 'MetaPhys': + maml_train(tepoch, model, inner_criterion, outer_criterion, inner_optimizer,optimizer, meta_params["num_adapt_steps"]) + else: + model.eval() + running_loss = 0.0 + with torch.no_grad(): + for inputs, target in tepoch: + tepoch.set_description(f"Validation") + outputs = model(inputs) + loss = criterion(outputs, target) + running_loss += loss.item() + tepoch.set_postfix(loss=running_loss / params["train_batch_size"]) + if min_val_loss > running_loss: # save the train model + min_val_loss = running_loss + checkpoint = {'Epoch': epoch, + 'state_dict': model.state_dict(), + 'optimizer': optimizer.state_dict()} + torch.save(checkpoint, params["checkpoint_path"] + model_params["name"] + "/" + + params["dataset_name"] + "_" + str(epoch) + "_" + + str(min_val_loss) + '.pth') if epoch + 1 == hyper_params["epochs"] or epoch % 3 == 0: if __TIME__ and epoch == 0: diff --git a/meta_params.json b/meta_params.json index effdd16..006c0fc 100644 --- a/meta_params.json +++ b/meta_params.json @@ -8,7 +8,7 @@ }, "params": { - "save_root_path": "/media/hdd1/yj/dataset/", + "save_root_path": "/media/hdd1/yj/dataset1/", "data_root_path": "/media/hdd1/", "dataset_name": "UBFC", "checkpoint_path" : "/media/hdd1/yj/checkpoint/", @@ -51,7 +51,7 @@ "meta_params": { "num_shots" : 6, - "num_test_shots" : 6, + "num_test_shots" : 2, "fs" : 30, "fs_comment" : "sampling rate of dataset", "unsupervised" : 0, diff --git a/nets/models/MetaPhys.py b/nets/models/MetaPhys.py index 4cced90..efce773 100644 --- a/nets/models/MetaPhys.py +++ b/nets/models/MetaPhys.py @@ -4,6 +4,7 @@ from nets.models.sub_models.LinearModel import LinearModel from nets.models.sub_models.MotionModel import MotionModel_TS +import higher class TSCAN(torch.nn.Module): def __init__(self): @@ -38,3 +39,73 @@ def forward(self, inputs): def get_attention_mask(self): return self.attention_mask1, self.attention_mask2 + +def maml_train(tepoch, model, inner_criterion, outer_criterion, inner_optimizer,optimizer, num_adapt_steps): + model.train() + + for batch in tepoch: + tepoch.set_description(f"Train Epoch ") + + batch['train'][0] = batch['train'][0].view(1, -1, 6, 36, 36) + batch['test'][0] = batch['test'][0].view(1, -1, 6, 36, 36) + batch['train'][1] = batch['train'][1].view(1, -1, 1) + batch['test'][1] = batch['test'][1].view(1, -1, 1) + + inputs, targets = batch['train'] + test_inputs, test_targets = batch['test'] + + test_losses = [] + optimizer.zero_grad() + for task_idx, (input, target, test_input, test_target) in enumerate( + zip(inputs, targets, test_inputs, test_targets)): + with higher.innerloop_ctx(model, inner_optimizer, copy_initial_weights=False) as (fmodel, diffopt): + for inner_step in range(num_adapt_steps): + inner_loss = inner_criterion(fmodel(input), target) + diffopt.step(inner_loss) + test_logit = fmodel(test_input) + test_loss = outer_criterion(test_logit, test_target) + test_losses.append(test_loss.detach()) + test_loss.backward() + + optimizer.step() + losses = sum(test_losses) / len(tepoch) + tepoch.set_postfix(loss=losses) + +def maml_val(tepoch, model, inner_criterion, outer_criterion, inner_optimizer, num_adapt_steps): + model.train() + test_losses = [] + for batch in tepoch: + tepoch.set_description(f"Validation") + + batch['train'][0] = batch['train'][0].view(1, -1, 6, 36, 36) + batch['test'][0] = batch['test'][0].view(1, -1, 6, 36, 36) + batch['train'][1] = batch['train'][1].view(1, -1, 1) + batch['test'][1] = batch['test'][1].view(1, -1, 1) + + inputs, targets = batch['train'] + test_inputs, test_targets = batch['test'] + + for task_idx, (input, target, test_input, test_target) in enumerate( + zip(inputs, targets, test_inputs, test_targets)): + with higher.innerloop_ctx(model, inner_optimizer, copy_initial_weights=False) as (fmodel, diffopt): + for step in range(num_adapt_steps): + inner_loss = inner_criterion(fmodel(input), target) + diffopt.step(inner_loss) + test_logit = fmodel(test_input).detach() + test_loss = outer_criterion(test_logit, test_target) + test_losses.append(test_loss.detach()) + + losses = sum(test_losses) / len(tepoch) + tepoch.set_postfix(loss=losses) + ''' + if min_val_loss > test_losses: # save the train model + min_val_loss = test_losses + checkpoint = {'Epoch': epoch, + 'state_dict': model.state_dict(), + 'optimizer': optimizer.state_dict()} + + torch.save(checkpoint, params["checkpoint_path"] + model_params["name"] + "/" + + params["dataset_name"] + "_" + str(epoch) + "_" + + str(min_val_loss) + '.pth') + + ''' \ No newline at end of file diff --git a/utils/Meta_class_splitters.py b/utils/Meta_class_splitters.py deleted file mode 100644 index 4810ae9..0000000 --- a/utils/Meta_class_splitters.py +++ /dev/null @@ -1,360 +0,0 @@ -import torch -import numpy as np - -from collections import OrderedDict, defaultdict -from torchmeta.utils.data.task import Task, ConcatTask, SubsetTask -from torchmeta.transforms.utils import apply_wrapper - -__all__ = ['Splitter', 'ClassSplitter', 'WeightedClassSplitter'] - - -class Splitter(object): - def __init__(self, splits, random_state_seed): - self.splits = splits - self.random_state_seed = random_state_seed - self.seed(random_state_seed) - - def seed(self, seed): - self.np_random = np.random.RandomState(seed=seed) - - def get_indices(self, task): - if isinstance(task, ConcatTask): - indices = self.get_indices_concattask(task) - elif isinstance(task, Task): - indices = self.get_indices_task(task) - else: - raise ValueError('The task must be of type `ConcatTask` or `Task`, ' - 'Got type `{0}`.'.format(type(task))) - return indices - - def get_indices_task(self, task): - raise NotImplementedError('Method `get_indices_task` must be ' - 'implemented in classes inherited from `Splitter`.') - - def get_indices_concattask(self, task): - raise NotImplementedError('Method `get_indices_concattask` must be ' - 'implemented in classes inherited from `Splitter`.') - - def _get_class_indices(self, task): - class_indices = defaultdict(list) - if task.num_classes is None: # Regression task - class_indices['regression'] = range(len(task)) - else: - for index in range(len(task)): - sample = task[index] - if (not isinstance(sample, tuple)) or (len(sample) < 2): - raise ValueError('In order to split the dataset in train/' - 'test splits, `Splitter` must access the targets. Each ' - 'sample from a task must be a tuple with at least 2 ' - 'elements, with the last one being the target.') - class_indices[sample[-1]].append(index) - - if len(class_indices) != task.num_classes: - raise ValueError('The number of classes detected in `Splitter` ' - '({0}) is different from the property `num_classes` ({1}) ' - 'in task `{2}`.'.format(len(class_indices), - task.num_classes, task)) - - return class_indices - - def __call__(self, task): - indices = self.get_indices(task) - return OrderedDict([(split, SubsetTask(task, indices[split])) - for split in self.splits]) - - def __len__(self): - return len(self.splits) - - -class ClassSplitter_(Splitter): - def __init__(self, shuffle=True, num_samples_per_class=None, - num_train_per_class=None, num_test_per_class=None, - num_support_per_class=None, num_query_per_class=None, - random_state_seed=0): - """ - Transforms a dataset into train/test splits for few-shot learning tasks, - based on a fixed number of samples per class for each split. This is a - dataset transformation to be applied as a `dataset_transform` in a - `MetaDataset`. - - Parameters - ---------- - shuffle : bool (default: `True`) - Shuffle the data in the dataset before the split. - - num_samples_per_class : dict, optional - Dictionary containing the names of the splits (as keys) and the - corresponding number of samples per class in each split (as values). - If not `None`, then the arguments `num_train_per_class`, - `num_test_per_class`, `num_support_per_class` and - `num_query_per_class` are ignored. - - num_train_per_class : int, optional - Number of samples per class in the training split. This corresponds - to the number of "shots" in "k-shot learning". If not `None`, this - creates an item `train` for each task. - - num_test_per_class : int, optional - Number of samples per class in the test split. If not `None`, this - creates an item `test` for each task. - - num_support_per_class : int, optional - Alias for `num_train_per_class`. If `num_train_per_class` is not - `None`, then this argument is ignored. If not `None`, this creates - an item `support` for each task. - - num_query_per_class : int, optional - Alias for `num_test_per_class`. If `num_test_per_class` is not - `None`, then this argument is ignored. If not `None`, this creates - an item `query` for each task. - - random_state_seed : int, optional - seed of the np.RandomState. Defaults to '0'. - - Examples - -------- - >>> transform = ClassSplitter(num_samples_per_class={ - ... 'train': 5, 'test': 15}) - >>> dataset = Omniglot('data', num_classes_per_task=5, - ... dataset_transform=transform, meta_train=True) - >>> task = dataset.sample_task() - >>> task.keys() - ['train', 'test'] - >>> len(task['train']), len(task['test']) - (25, 75) - """ - self.shuffle = shuffle - - if num_samples_per_class is None: - num_samples_per_class = OrderedDict() - if num_train_per_class is not None: - num_samples_per_class['train'] = num_train_per_class - elif num_support_per_class is not None: - num_samples_per_class['support'] = num_support_per_class - if num_test_per_class is not None: - num_samples_per_class['test'] = num_test_per_class - elif num_query_per_class is not None: - num_samples_per_class['query'] = num_query_per_class - assert len(num_samples_per_class) > 0 - - self._min_samples_per_class = sum(num_samples_per_class.values()) - super(ClassSplitter_, self).__init__(num_samples_per_class, random_state_seed) - - def get_indices_task(self, task): - all_class_indices = self._get_class_indices(task) - indices = OrderedDict([(split, []) for split in self.splits]) - - for name, class_indices in all_class_indices.items(): - num_samples = len(class_indices) - if self.shuffle: - seed = (hash(task) + self.random_state_seed) % (2 ** 32) - dataset_indices = np.random.RandomState(seed).permutation(num_samples) - else: - dataset_indices = np.arange(num_samples) - - ptr = 0 - for split, num_split in self.splits.items(): - split_indices = dataset_indices[ptr:ptr + num_split] - if self.shuffle: - self.np_random.shuffle(split_indices) - indices[split].extend([class_indices[idx] for idx in split_indices]) - ptr += num_split - return indices - - def get_indices_concattask(self, task): - indices = OrderedDict([(split, []) for split in self.splits]) - cum_size = 0 - - for dataset in task.datasets: - num_samples = len(dataset) - if num_samples < self._min_samples_per_class: - raise ValueError('The number of samples for one class ({0}) ' - 'is smaller than the minimum number of samples per class ' - 'required by `ClassSplitter` ({1}).'.format(num_samples, - self._min_samples_per_class)) - - if self.shuffle: - seed = (hash(task) + self.random_state_seed) % (2 ** 32) - dataset_indices = np.random.RandomState(seed).permutation(num_samples) - else: - dataset_indices = np.arange(num_samples) - - ptr = 0 - for split, num_split in self.splits.items(): - split_indices = dataset_indices[ptr:ptr + num_split] - if self.shuffle: - self.np_random.shuffle(split_indices) - indices[split].extend(split_indices + cum_size) - ptr += num_split - cum_size += num_samples - return indices - - -class WeightedClassSplitter_(Splitter): - def __init__(self, shuffle=True, min_num_samples=1, max_num_samples=None, - weights=None, train_weights=None, test_weights=None, - support_weights=None, query_weights=None, - force_equal_per_class=False, random_state_seed=0): - """ - Transforms a dataset into train/test splits for few-shot learning tasks. - The number of samples per class is proportional to the number of samples - per class in the original dataset. This is a dataset transformation to - be applied as a `dataset_transform` in a `MetaDataset`. - - Parameters - ---------- - shuffle : bool (default: `True`) - Shuffle the data in the dataset before the split. - - min_num_samples : int or dict, optional (default: 1) - Minimum number of samples per class. - - max_num_samples : int or dict, optional - Maximum number of samples per class. - - weights : dict, optional - Dictionary containing the names of the splits (as keys) and the - corresponding proportions of samples per class in each split (as - values). If not `None`, then the arguments `train_weights`, - `test_weights`, `support_weights` and `query_weights` are ignored. - - train_weights : float, optional - Proportion of samples from each class in the training split. If not - `None`, this creates an item `train` for each task. - - test_weights : float, optional - Proportion of samples from each class in the training split. If not - `None`, this creates an item `test` for each task. - - support_weights : float, optional - Alias for `train_weights`. If `train_weights` is not `None`, then - this argument is ignored. If not `None`, this creates an item - `support` for each task. - - query_weights : float, optional - Alias for `test_weights`. If `test_weights` is not `None`, then this - argument is ignored. If not `None`, this creates an item `query` for - each task. - - force_equal_per_class : bool (default: `False`) - If `True`, then the number of samples per class is equal for each - class; this is then proportional to the number of samples in the - class with the minimum number of samples. - - random_state_seed : int, optional - seed of the np.RandomState. Defaults to '0'. - """ - self.shuffle = shuffle - self.force_equal_per_class = force_equal_per_class - - if weights is None: - weights = OrderedDict() - if train_weights is not None: - weights['train'] = train_weights - elif support_weights is not None: - weights['support'] = support_weights - if test_weights is not None: - weights['test'] = test_weights - elif query_weights is not None: - weights['query'] = query_weights - assert len(weights) > 0 - assert sum(weights.values()) <= 1. - - if (min_num_samples is None) or isinstance(min_num_samples, int): - if min_num_samples is None: - min_num_samples = 0 - self.min_num_samples = OrderedDict([(split, min_num_samples) - for split in weights]) - elif isinstance(min_num_samples, dict): - self.min_num_samples = OrderedDict(min_num_samples) - else: - raise NotImplementedError('Argument `min_num_samples` in ' - '`WeightedClassSplitter` must be of type `dict` or `int`. Got ' - 'type `{0}`.'.format(type(min_num_samples))) - - if max_num_samples is None: - self.max_num_samples = None - elif isinstance(max_num_samples, int): - self.max_num_samples = OrderedDict([(split, max_num_samples) - for split in weights]) - elif isinstance(max_num_samples, dict): - self.max_num_samples = OrderedDict(max_num_samples) - else: - raise NotImplementedError('Argument `max_num_samples` in ' - '`WeightedClassSplitter` must be of type `dict` or `int`. Got ' - 'type `{0}`.'.format(type(min_num_samples))) - - self._min_samples_per_class = sum(self.min_num_samples.values()) - super(WeightedClassSplitter_, self).__init__(weights, random_state_seed) - - def get_indices_task(self, task): - all_class_indices = self._get_class_indices(task) - indices = OrderedDict([(split, []) for split in self.splits]) - - min_samples = min([len(class_indices) for class_indices - in all_class_indices.values()]) - if min_samples < self._min_samples_per_class: - raise ValueError('The smallest number of samples in a class ({0}) ' - 'is smaller than the minimum number of samples per class ' - 'required by `WeightedClassSplitter` ({1}).'.format( - min_samples, self._min_samples_per_class)) - - for class_indices in all_class_indices.values(): - num_samples = (min_samples if self.force_equal_per_class - else len(class_indices)) - if self.shuffle: - seed = (hash(task) + self.random_state_seed) % (2 ** 32) - dataset_indices = np.random.RandomState(seed).permutation(num_samples) - else: - dataset_indices = np.arange(num_samples) - - ptr = 0 - for split, weight in self.splits.items(): - num_split = max(self.min_num_samples[split], int(weight * num_samples)) - if self.max_num_samples is not None: - num_split = min(self.max_num_samples[split], num_split) - split_indices = dataset_indices[ptr:ptr + num_split] - if self.shuffle: - self.np_random.shuffle(split_indices) - indices[split].extend([class_indices[idx] for idx in split_indices]) - ptr += num_split - - return indices - - def get_indices_concattask(self, task): - indices = OrderedDict([(split, []) for split in self.splits]) - cum_size = 0 - - min_samples = min([len(dataset) for dataset in task.datasets]) - if min_samples < self._min_samples_per_class: - raise ValueError('The smallest number of samples in a class ({0}) ' - 'is smaller than the minimum number of samples per class ' - 'required by `WeightedClassSplitter` ({1}).'.format( - min_samples, self._min_samples_per_class)) - - for dataset in task.datasets: - num_samples = (min_samples if self.force_equal_per_class - else len(dataset)) - if self.shuffle: - seed = (hash(task) + self.random_state_seed) % (2 ** 32) - dataset_indices = np.random.RandomState(seed).permutation(num_samples) - else: - dataset_indices = np.arange(num_samples) - - ptr = 0 - for split, weight in self.splits.items(): - num_split = max(self.min_num_samples, int(weight * num_samples)) - split_indices = dataset_indices[ptr:ptr + num_split] - if self.shuffle: - self.np_random.shuffle(split_indices) - indices[split].extend(split_indices + cum_size) - cum_size += num_samples - - return indices - - -def ClassSplitter(task=None, *args, **kwargs): - return apply_wrapper(ClassSplitter_(*args, **kwargs), task) - -def WeightedClassSplitter(task=None, *args, **kwargs): - return apply_wrapper(WeightedClassSplitter_(*args, **kwargs), task) diff --git a/utils/dataset_preprocess.py b/utils/dataset_preprocess.py index c6e5896..419237a 100755 --- a/utils/dataset_preprocess.py +++ b/utils/dataset_preprocess.py @@ -56,61 +56,6 @@ def preprocessing(save_root_path: str = "/media/hdd1/dy_dataset/", test_file.close() -def Meta_preprocessing(save_root_path: str = "/media/hdd1/dy_dataset/", - model_name: str = "DeepPhys", - data_root_path: str = "/media/hdd1/", - dataset_name: str = "UBFC", - train_ratio: float = 0.8): - """ - :param save_root_path: save file destination path - :param model_name: select preprocessing method - :param data_root_path: data set root path - :param dataset_name: data set name(ex. UBFC, COFACE) - :param train_ratio: data split [ train ratio : 1 - train ratio] - :return: - """ - dataset_root_path = data_root_path + dataset_name - - manager = multiprocessing.Manager() - return_dict = manager.dict() - - data_list = [data for data in os.listdir(dataset_root_path) if data.__contains__("subject")] - - process = [] - - # multiprocessing - for index, data_path in enumerate(data_list): - print('datapreprocessing..',data_path) - - proc = multiprocessing.Process(target=preprocess_Dataset, - args=(dataset_root_path + "/" + data_path, True, model_name, return_dict)) - process.append(proc) - proc.start() - - for proc in process: - proc.join() - - for index, data_path in enumerate(return_dict.keys()): - video = return_dict[data_path]['preprocessed_video'] - label = return_dict[data_path]['preprocessed_label'] - - data_len = len(label) //8 # 1개의 데이터를 8개로 - for i in range(8): - n_video = video[data_len * i:data_len * (i + 1)] - n_label = label[data_len * i:data_len * (i + 1)] - - dir = save_root_path + model_name + '/'+dataset_name+'/'+data_path - if not os.path.exists(dir): - os.makedirs(dir) - - train_file = h5py.File(dir+"/{}_{}.hdf5".format(data_path, i), "w") - train_file.create_dataset('preprocessed_video', data=n_video) - train_file.create_dataset('preprocessed_label', data=n_label) - #print(train_file["preprocessed_video"]) - #print(train_file["preprocessed_label"]) - train_file.close() - - def preprocess_Dataset(path, flag, model_name, return_dict): """ :param path: dataset path