added tCNN models

LCTM/datasets.py

@@ -7,16 +7,24 @@
 class Dataset:
     name = ""
     features = ""
+    n_classes = None
+    n_features = None
+    sep_splits = False
 
-    def __init__(self, base_dir, features):
+    def __init__(self, base_dir, features, sep_splits=False):
         self.base_dir = base_dir + "{}/".format(self.name)
         self.features = features
+        self.sep_splits = sep_splits
 
 
-    def get_files(dataset):
-        files_features = np.sort(os.listdir(dataset.dir_labels))
+    def get_files(self, idx_task=None):
+        if self.sep_splits:
+            files_features = np.sort(os.listdir(self.dir_labels+"/Split_{}/".format(idx_task)))
+        else:
+            files_features = np.sort(os.listdir(self.dir_labels))
+
         files_features = [f for f in files_features if f.find(".mat")>=0]
-        files_features = [f.replace(".mat", "") for f in files_features]
+        # files_features = [f.replace(".mat", "") for f in files_features]
         return files_features
 
     def load_split(self, idx_task):
@@ -28,11 +36,15 @@ def load_split(self, idx_task):
         file_test = [f.strip() for f in file_test]
 
         # Format the train/test split names
-        files_features = self.get_files()
+        files_features = self.get_files(idx_task)
 
         # Load data
-        Y_all = [ sio.loadmat( "{}{}.mat".format(self.dir_labels,f) )["Y"].ravel() for f in files_features]
-        X_all = [ sio.loadmat( "{}{}.mat".format(self.dir_features,f) )["X"].astype(np.float64) for f in files_features]
+        if self.sep_splits:
+            Y_all = [ sio.loadmat( "{}/Split_{}/{}".format(self.dir_labels,idx_task,f) )["Y"].ravel() for f in files_features]
+            X_all = [ sio.loadmat( "{}/Split_{}/{}".format(self.dir_features,idx_task, f) )["X"].astype(np.float64) for f in files_features]
+        else:
+            Y_all = [ sio.loadmat( "{}{}".format(self.dir_labels,f) )["Y"].ravel() for f in files_features]
+            X_all = [ sio.loadmat( "{}/{}".format(self.dir_features, f) )["X"].astype(np.float64) for f in files_features]
 
         # Make sure labels are sequential
         Y_all = utils.remap_labels(Y_all)
@@ -44,6 +56,9 @@ def load_split(self, idx_task):
         if X_all[0].shape[0] > X_all[0].shape[1]:
             X_all = [x.T for x in X_all]
 
+        self.n_classes = len(np.unique(np.hstack(Y_all)))
+        self.n_features = X_all[0].shape[0]
+
         # ------------Train/test Splits---------------------------
         # Split data/labels into train/test splits
         fid2idx = self.fix2idx(files_features)
@@ -59,27 +74,44 @@ class JIGSAWS(Dataset):
     n_splits = 7
     name = "JIGSAWS"
 
-    def __init__(self, base_dir, features):
-        Dataset.__init__(self, base_dir, features)
+    def __init__(self, base_dir, features, sep_splits=False):
+        Dataset.__init__(self, base_dir, features, sep_splits)
 
         # Setup directory and filenames
         self.dir_labels = os.path.expanduser(self.base_dir+"labels/sequences/Suturing/")
         self.dir_features = os.path.expanduser(self.base_dir+"features/{}/".format(self.features))
 
     def fix2idx(self, files_features):
-        return {files_features[i]:i for i in range(len(files_features))}
+        if files_features[0].find(".mat"):
+            return {files_features[i].replace(".mat",""):i for i in range(len(files_features))}
+        else:
+            return {files_features[i]:i for i in range(len(files_features))}
 
 class Salads(Dataset):
     n_splits = 5
     name = "50Salads"
 
-    def __init__(self, base_dir, features):
-        Dataset.__init__(self, base_dir, features)
+    def __init__(self, base_dir, features, sep_splits=False):
+        Dataset.__init__(self, base_dir, features, sep_splits)
 
         # Setup directory and filenames
         self.dir_features = os.path.expanduser(self.base_dir+"features/{}/Split_1/".format(self.features))
         self.dir_labels = os.path.expanduser(self.base_dir+"features/{}/Split_1/".format(self.features))
 
     def fix2idx(self, files_features):
-        return {files_features[i].strip("rgb-").strip(".avi"):i for i in range(len(files_features))}        
+        return {files_features[i].replace("rgb-","").replace(".mat","").replace(".avi",""):i for i in range(len(files_features))}        
+
+class EndoVis(Dataset):
+    n_splits = 7
+    name = "EndoVis"
+
+    def __init__(self, base_dir, features, sep_splits=False):
+        Dataset.__init__(self, base_dir, features, sep_splits)
+
+        # Setup directory and filenames
+        self.dir_features = os.path.expanduser(self.base_dir+"features/{}/".format(self.features))
+        self.dir_labels = os.path.expanduser(self.base_dir+"features/{}/".format(self.features))
+
+    def fix2idx(self, files_features):
+        return {files_features[i].replace(".mat",""):i for i in range(len(files_features))}
 

LCTM/dtw.py

@@ -1,7 +1,7 @@
 
 
 import numpy as np
-from numba import float64, jit, int16, boolean
+from numba import float64, jit, int16, boolean, autojit
 import scipy
 import sklearn.neighbors as sn
 from LCTM import utils
@@ -23,15 +23,39 @@
 
 #     cost = cost_U + cost_L
 
-@jit("float32(float32[:,:], float32[:,:], float32)")
-def DTW(x, y, max_value=np.inf):
+@jit("int32(float32[:,:])")
+def _traceback(D):
+    n_ref, n_in = D.shape
+    correspondences = np.zeros(n_in, np.int)
+    correspondences[-1] = n_ref-1
+
+    c = n_ref-1
+    i = n_in-1
+    while i > 0 and c > 0:
+        a = np.argmin([
+                       D[c-1, i-1],
+                       D[c-1, i],
+                       D[c, i-1]
+                       ])
+        if a==0 or a==1:
+            c -= 1
+        if a==0 or a==2:
+            i -= 1
+        correspondences[i] = c
+
+    return correspondences
+
+# @jit("(float32, int32[:])(float32[:,:], float32[:,:], float32)")
+@autojit
+def DTW(x, y, max_value=np.inf, output_correspondences=False):
+    # Should be of shape FxT
     Tx = x.shape[1]
     Ty = y.shape[1]
 
     cost = np.zeros([Tx, Ty], np.float32) + np.inf
     # Compute first row
-    # cost[0,:] = ((x[:,0][:,None]-y)**2).sum(0).cumsum(0)
-    cost[0,:] = ((x[:,0][:,None]!=y)).sum(0).cumsum(0)
+    cost[0,:] = ((x[:,0][:,None]-y)**2).sum(0).cumsum(0)
+    # cost[0,:] = ((x[:,0][:,None]!=y)).sum(0).cumsum(0)
 
     # Compute rest of the rows
     tx = 1
@@ -42,8 +66,8 @@ def DTW(x, y, max_value=np.inf):
             top = cost[tx-1, ty]
             left = cost[tx, ty-1]
 
-            # current = np.sum((x[:,tx]-y[:,ty])**2)
-            current = np.sum((x[:,tx]!=y[:,ty]))
+            current = np.sum((x[:,tx]-y[:,ty])**2)
+            # current = np.sum((x[:,tx]!=y[:,ty]))
             cost[tx, ty] = min(topleft, left, top) + current
 
             # if cost[tx, ty] > max_value:
@@ -53,9 +77,13 @@ def DTW(x, y, max_value=np.inf):
             ty += 1
         tx += 1
 
-    # cost[cost==inf] = 0
+    if output_correspondences:
+        c = _traceback(cost)
+        return cost[-1,-1], c
+    else:
+        return cost[-1,-1]
+
 
-    return cost[-1,-1]
 
 # def DTW(x, y, max_value=np.inf):
 #     Tx = x.shape[1]

LCTM/learn.py

@@ -28,11 +28,16 @@ def subgradient_descent(model, X, Y, n_iter=100, C=1., pretrain=True, verbose=Tr
         np.random.seed(1234)
 
     n_samples = len(X)
+
+    # Check that Xi is of size FxT
+    if X[0].shape[0] > X[0].shape[0]:
+        X = [x.T for x in X]
 
     # if weights haven't been set yet then initialize
     if model.n_classes is None:
         model.n_features = X[0].shape[0]
         model.n_classes = np.max(list(map(np.max, Y)))+1
+        model.max_segs = utils.max_seg_count(Y)
         model.ws.init_weights(model)
 
         if pretrain:

LCTM/models.py

@@ -65,15 +65,22 @@ def n_nodes(model):
             n_nodes = model.n_classes
         return n_nodes
 
-    def predict(model, Xi, Yi=None, is_training=False, output_latent=False):
+    def predict(model, Xi, Yi=None, is_training=False, output_latent=False, inference=None):
         # This function is applicable to normal and latent models
         if type(Xi) is list:
             out = []
             for i in range(len(Xi)):
                 Yi_ = None if Yi is None else Yi[i]
-                out += [model.predict(Xi[i], Yi=Yi_, is_training=is_training, output_latent=output_latent)]
+                out += [model.predict(Xi[i], Yi_, is_training, output_latent, inference)]
             return out
 
+        # Check that Xi is of size FxT
+        if Xi.shape[0] > Xi.shape[0]:
+            Xi = Xi.T
+
+        if Yi is not None:
+            assert Xi.shape[1]==Yi.shape[0], "Error: Xi and Yi are of shapes {} and {}".format(Xi.shape[1],Yi.shape[0])
+
         _, n_timesteps = Xi.shape
         n_nodes = model.n_nodes
 
@@ -95,15 +102,16 @@ def predict(model, Xi, Yi=None, is_training=False, output_latent=False):
             score = reduce_latent_states(score, model.n_latent, model.n_classes)
 
         # Get predictions
-        if model.inference_type is "framewise":
+        inference_type = model.inference_type if inference is None else inference
+        if inference_type is "framewise":
             path = score.argmax(0)
 
-        elif model.inference_type is "filtered":
+        elif inference_type is "filtered":
             assert hasattr(model, "filter_len"), "filter_len must be set"
             path = score.argmax(0)
             path = nd.median_filter(path, model.filter_len)
 
-        elif model.inference_type is "segmental":
+        elif inference_type is "segmental":
             assert hasattr(model, "max_segs"), "max_segs must be set"
             # Check if there is a segmental pw.pairwise term:
             # has_seg_pw = any([type(p) is pw.segmental_pairwise for p in model.potentials.values()])

LCTM/utils.py

@@ -16,18 +16,45 @@ def imshow_(x):
 	plt.axis("tight")
 
 # ------------- Data -------------
+# def mask_data(X, Y):
+# 	max_len = np.max([x.shape[0] for x in X])
+# 	X_ = np.zeros([len(X), max_len, X[0].shape[1]])
+# 	Y_ = np.zeros([len(X), max_len, Y[0].shape[1]])
+# 	mask = np.zeros([len(X), max_len])
+# 	for i in range(len(X)):
+# 		l = X[i].shape[0]
+# 		X_[i,:l] = X[i]
+# 		Y_[i,:l] = Y[i]
+# 		mask[i,:l] = 1
+# 	return X_, Y_, mask[:,:,None]
+
 def mask_data(X, Y):
-	max_len = np.max([x.shape[0] for x in X])
-	X_ = np.zeros([len(X), max_len, X[0].shape[1]])
-	Y_ = np.zeros([len(X), max_len, Y[0].shape[1]])
+	max_len = np.max([x.shape[1] for x in X])
+	X_ = np.zeros([len(X), X[0].shape[0], max_len])
+	Y_ = np.zeros([len(X), Y[0].shape[0], max_len])
 	mask = np.zeros([len(X), max_len])
 	for i in range(len(X)):
-		l = X[i].shape[0]
-		X_[i,:l] = X[i]
-		Y_[i,:l] = Y[i]
+		l = X[i].shape[1]
+		X_[i,:,:l] = X[i]
+		Y_[i,:,:l] = Y[i]
 		mask[i,:l] = 1
 	return X_, Y_, mask[:,:,None]
 
+# Unmask data
+def unmask(X, M):
+	if X.ndims==1 or (X[0].shape[0] > X[0].shape[1]):
+		return [X[i][M[i].flatten()>0] for i in range(len(X))]
+	else:
+		return [X[i][:,M[i].flatten()>0] for i in range(len(X))]
+
+def match_lengths(X,Y):
+	# Check lengths of data and labels match
+	for i in range(len(Y)):
+		length = min(X[i].shape[0], Y[i].shape[0])
+		X[i] = X[i][:length]
+		Y[i] = Y[i][:length]
+	return X, Y
+
 def remap_labels(Y_all):
 	# Map arbitrary set of labels (e.g. {1,3,5}) to contiguous sequence (e.g. {0,1,2})
 	ys = np.unique([np.hstack([np.unique(Y_all[i]) for i in range(len(Y_all))])])

eval_ICRA.py

@@ -15,9 +15,11 @@
 from LCTM import metrics
 
 # Directories and filename
-base_dir = expanduser("~/Data/")
-# base_dir = "/home/colin/data/"
-save_dir = expanduser("~/Data/Results/")
+base_dir = expanduser("~/data/")
+save_dir = expanduser("~/data/Results/")
+# base_dir = expanduser("~/Data/")
+# save_dir = expanduser("~/Data/Results/")
+
 
 # ------------------------------------------------------------------
 # If running from command line get split index number