Init LRP with embedding layer

src/IMDB_dataset/imdb_preprocess.py

@@ -61,11 +61,12 @@ def pad_sequences(trainX, validX, testX, maxlen=None, dtype='int32', padding='po
        if len(s) < maxlen:
           while not len(s)==maxlen:
                 s.append(pads)
+
     for s in testX:
        if len(s) < maxlen:
           while not len(s)==maxlen:
                 s.append(pads)           
-
+    
     return trainX,validX,testX,maxlen
 
 
@@ -251,11 +252,12 @@ def preprocess_IMDBdata(seed,filenames,n_words=None,dictionary=None,test_size=0.
     trainX,validX,testX,filenames_train,filenames_valid,filenames_test,test_dict = extract_features(filenames,seed,test_size,save_test,n_words,dictionary)
 #    extract_features_w2v(filenames,seed,test_size,save_test=None)
 
-    trainX,validX,testX,maxlen = pad_sequences(trainX, validX, testX, value=0.)
+    trainX,validX,testX,maxlen = pad_sequences(trainX, validX,testX, value=0.)
     trainX = np.array(trainX)
     validX = np.array(validX)
     testX  = np.array(testX)
 
+    print(testX)
     trainY,validY,testY = extract_labels(filenames_train,filenames_valid,filenames_test)
 
     return trainX,validX,testX,trainY,validY,testY,filenames_train,filenames_valid,filenames_test,maxlen,test_dict

src/dynamic_lstm_TF.py

@@ -12,7 +12,7 @@
     - http://ai.stanford.edu/~amaas/data/sentiment/
 """
 from __future__ import division, print_function, absolute_import
-from IMDB_dataset.textData_cluster import filenames
+from IMDB_dataset.textData import filenames
 from parameter_persistence import export_serial_model,export_serial_lstm_data
 from sacred.observers import FileStorageObserver
 import IMDB_dataset.imdb_preprocess as imdb_pre
@@ -52,14 +52,15 @@ def config():
     save_path = "./sacred_models/"
     tensorboard_dir = "./sacred_models/tf_logs/"
     run_id = "runID_newOutput"
-    n_words = 10000
-    dictionary = "/home/icha/tRustNN/imdb_dict.pickle"
+    n_words = 10000 #89527 
+    dictionary = "/home/yannis/Desktop/tRustNN/imdb_dict.pickle"
     embedding_dim = 300
     ckp_path = None #"./sacred_models/ckp/"
     internals = "all"    
     save_mode = "pickle"
-    n_epoch = 10
+    n_epoch = 2
     test_size = 0.05 # -1 for whole test set
+    embedding_layer = 1
 
     #Dictionary describing the architecture of the network
     net_arch = collections.OrderedDict()
@@ -78,13 +79,17 @@ def config():
 
 
 
-def build_network(net_arch,net_arch_layers,tensorboard_verbose,sequence_length,embedding_dim,tensorboard_dir,batch_size,ckp_path=None):
+def build_network(net_arch,net_arch_layers,tensorboard_verbose,sequence_length,embedding_dim,tensorboard_dir,batch_size,n_words,embedding_layer,ckp_path=None):
 
-    # Network building
-    net = tflearn.input_data([None,sequence_length])
-    net = tflearn.embedding(net, input_dim=10000, output_dim=10)
     layer_outputs = dict()
-    prev_incoming = net
+
+    # Network building
+    if embedding_layer:
+        net = tflearn.input_data([None,sequence_length]) #embedding_dim
+        ebd_output = tflearn.embedding(net, input_dim=n_words, output_dim=embedding_dim, name='embedding')
+        n = "embedding_output"
+        layer_outputs[n] = ebd_output
+        prev_incoming = ebd_output
 
     for k in range(len(net_arch_layers)):
         key = net_arch_layers[k]
@@ -126,17 +131,19 @@ def build_network(net_arch,net_arch_layers,tensorboard_verbose,sequence_length,e
 
 
 @ex.automain
-def train(seed,net_arch,net_arch_layers,save_path,n_epoch,tensorboard_verbose,show_metric,batch_size,run_id,db,n_words,dictionary,embedding_dim,tensorboard_dir,ckp_path,internals,save_mode,test_size):
+def train(seed,net_arch,net_arch_layers,save_path,n_epoch,tensorboard_verbose,show_metric,batch_size,run_id,db,n_words,dictionary,embedding_dim,tensorboard_dir,ckp_path,embedding_layer,internals,save_mode,test_size):
 
     save_dir = save_path+run_id+"/"
     if not os.path.exists(save_dir):
         os.makedirs(save_dir)
-
-
+    """
     print("Extracting features...")
     #Train, valid and test sets. Have to return filenames_test as we have now shuffled them
     trainX,validX,testX,trainY,validY,testY,filenames_train,filenames_valid,filenames_test_sfd,maxlen,test_dict = imdb_pre.preprocess_IMDBdata(seed=seed,filenames=filenames,n_words=n_words,dictionary=dictionary,test_size=test_size,save_test="save_test")
-
+    """
+    with open('trainValidtestNew.pickle','rb') as handle:
+        (trainX,validX,testX,trainY,validY,testY,filenames_train,filenames_valid,filenames_test_sfd,maxlen,test_dict) = pickle.load(handle)
+    """
     d = test_dict        
     if save_mode=="pickle":
         with open(save_dir+"test_data_input.pickle", "wb") as f:
@@ -145,21 +152,20 @@ def train(seed,net_arch,net_arch_layers,save_path,n_epoch,tensorboard_verbose,sh
         with open(save_dir+"test_data_input.json", "w") as f:
             json.dump(d, f)
     print("Exported test data dictionary...")
-    """ 
-    
-    with open('trainValidtest.pickle','rb') as handle:
-        (trainX,validX,testX,trainY,validY,testY,filenames_train,filenames_valid,filenames_test_sfd) = pickle.load(handle)
     """
+
+#    with open('trainValidtestNew.pickle','wb') as handle:
+#        pickle.dump((trainX,validX,testX,trainY,validY,testY,filenames_train,filenames_valid,filenames_test_sfd,maxlen,test_dict),handle)
 
     print("Training model...")
 
-    model, layer_outputs = build_network(net_arch,net_arch_layers,tensorboard_verbose,trainX.shape[1],embedding_dim,tensorboard_dir,batch_size,ckp_path)
+    model, layer_outputs = build_network(net_arch,net_arch_layers,tensorboard_verbose,trainX.shape[1],embedding_dim,tensorboard_dir,batch_size,n_words,embedding_layer,ckp_path)
     model.fit(trainX, trainY, validation_set=(validX, validY), n_epoch=n_epoch, show_metric=show_metric, batch_size=batch_size)
-
+    """
     print("Evaluating trained model on test set...")
-    score = model.evaluate(testX,testY,batch_size=maxlen)
+    score = model.evaluate(testX,testY)
     print("Accuracy on test set: %0.4f%%" % (score[0] * 100))
-
+    """
 
     #Save model to json format
     export_serial_model(model,net_arch_layers,save_dir)
@@ -174,18 +180,25 @@ def train(seed,net_arch,net_arch_layers,save_path,n_epoch,tensorboard_verbose,sh
     input_files = filenames_valid
     export_serial_lstm_data(model,layer_outputs,feed,input_files,internals,save_dir+"valid_")
     """
+
     feed = testX
     input_files = filenames_test_sfd
+    """
     export_serial_lstm_data(model,layer_outputs,feed,input_files,internals,save_dir+"test_",save_mode=save_mode)
-    print("Exported internals...")
     
+    print("Exported internals...")
+    """
     #Delete part that creates problem in restoring model - should still be able to evaluate, but tricky for continuing training
     del tf.get_collection_ref(tf.GraphKeys.TRAIN_OPS)[:]
     model.save(save_dir+"tf_model.tfl")
     print("Saved model...")    
-    
+
     predicted_tgs = model.predict_label(feed)
-#    LRP = lrp.lrp_full(model,input_files,net_arch,net_arch_layers,save_dir+"test_data_input."+save_mode,save_dir+"test_model_internals_fc."+save_mode,save_dir+"test_model_internals_lstm_hidden."+save_mode,save_dir+"test_model_internals_lstm_states."+save_mode,eps=0.001,delta=0.0,save_dir=save_dir,lstm_actv1=expit,lstm_actv2=np.tanh,topN=5,debug=False,predictions=predicted_tgs)
+
+    LRP = lrp.lrp_full(model,embedding_layer,n_words,input_files,net_arch,net_arch_layers,save_dir+"test_data_input."+save_mode,save_dir+"test_model_internals_fc."+save_mode,save_dir+"test_model_internals_lstm_hidden."+save_mode,save_dir+"test_model_internals_lstm_states."+save_mode,save_dir+"test_model_internals_ebd."+save_mode,eps=0.001,delta=0.0,save_dir=save_dir,lstm_actv1=expit,lstm_actv2=np.tanh,topN=5,debug=False,predictions=predicted_tgs)
+
+
+
     with open(save_dir+"lstm_predictions.pickle","wb") as handle:
         pickle.dump(predicted_tgs,handle)
     print("Finished with LRP and related data...now exiting...")

src/lrp.py

@@ -201,6 +201,24 @@ def get_gates_out_t(in_concat,b,i_arr,f_arr,g_arr,o_arr,d,lstm_actv1,lstm_actv2)
      return i_t,g_t,f_t
 
 
+def lrp_embedding(model,emb_name,n_words,feed,lstm_first_input,lrp_lstm,eps,delta,debug):
+#first_lstm_output : the lstm layer that connects to the embedding
+
+        layer = tflearn.variables.get_layer_variables_by_name(emb_name)
+        W_ebd = model.get_weights(layer[0])
+        zi = feed
+        zj = lstm_first_input #(300,)
+        W = W_ebd
+        b = np.zeros((zj.shape[0]))
+        Rout = lrp_lstm 
+        N = n_words
+
+        lrp_ebd = lrp_linear(zi, W, b, zj, Rout, N, eps, delta, debug)
+
+        return lrp_ebd
+
+
+
 
 def lrp_fullyConnected(model,fc_name,last_lstm_output,fc_out,lrp_mask,d,T,classes,eps,delta,debug):
 #last_lstm_output : the lstm layer that connects to the fully connected
@@ -271,50 +289,62 @@ def lrp_lstm(model,layer_name,feed,T,d,lrp_fc,lstm_hidden,lstm_cell,lstm_actv1,l
         return lstm_lrp_x,(lstm_lrp_h,lstm_lrp_g,lstm_lrp_c)
 
 
-def load_intermediate_outputs(input_filename,fc_json,lstm_hidden_json,lstm_cell_json,layer_name=None):
+def load_intermediate_outputs(input_filename,embedding_json,fc_json,lstm_hidden_json,lstm_cell_json,layer_name=None):
 #layer_name is currently not needed - for networks with more layers we will need it, as the json structure will change
 
 
      keys_hidden,data_hidden = data_format.get_data(lstm_hidden_json)
      keys_cell,data_cell = data_format.get_data(lstm_cell_json)
      keys_fc,data_fc = data_format.get_data(fc_json)
-
+     keys_ebd,data_ebd = data_format.get_data(embedding_json)
+
      lstm_hidden = data_hidden[input_filename]
      lstm_cell   = data_cell[input_filename]
      fc_out      = data_fc[input_filename]
+     embedding_output_data = data_ebd[input_filename]
 
      d = lstm_cell.shape[1]
 
-     return fc_out,lstm_hidden,lstm_cell,d
+     return fc_out,lstm_hidden,lstm_cell,embedding_output_data,d
 
 
 
-def lrp_single_input(model,layer_names,input_filename,single_input_data,eps,delta,fc_json,lstm_hidden_json,lstm_cell_json,target_class,T,classes=2,lstm_actv1=expit,lstm_actv2=np.tanh,debug=False):
+def lrp_single_input(model,embedding_layer,n_words,layer_names,input_filename,single_input_data,eps,delta,fc_json,lstm_hidden_json,lstm_cell_json,ebd_json,target_class,T,classes=2,lstm_actv1=expit,lstm_actv2=np.tanh,debug=False):
 
 
     with model.session.as_default():
 
         lrp_mask = np.zeros((classes))
         lrp_mask[target_class] = 1.0
 
-        fc_out,lstm_hidden,lstm_cell,d = load_intermediate_outputs(input_filename,fc_json,lstm_hidden_json,lstm_cell_json,layer_name=None)
+        fc_out,lstm_hidden,lstm_cell,embedding_output_data,d = load_intermediate_outputs(input_filename,ebd_json,fc_json,lstm_hidden_json,lstm_cell_json,layer_name=None)
 
         #LRP through fc layer
         fc_name = "fc"
         lrp_fc = lrp_fullyConnected(model,fc_name,lstm_hidden,fc_out,lrp_mask,d,T,classes,eps,delta,debug)
 
-        #LRP through lstm layer
-        lstm_name = "lstm"        
-        feed = single_input_data
-        lstm_lrp_x,(lstm_lrp_h,lstm_lrp_g,lstm_lrp_c) = lrp_lstm(model,lstm_name,feed,T,d,lrp_fc,lstm_hidden,lstm_cell,lstm_actv1,lstm_actv2,eps,delta,debug)
+        #LRP through embedding layer if needed
+        if embedding_layer:
+            lstm_name = "lstm"        
+            feed = embedding_output_data
+            lstm_lrp_x,(lstm_lrp_h,lstm_lrp_g,lstm_lrp_c) = lrp_lstm(model,lstm_name,feed,T,d,lrp_fc,lstm_hidden,lstm_cell,lstm_actv1,lstm_actv2,eps,delta,debug)
+
+            emb_name = "embedding"
+            feed = single_input_data
+            lrp_input = lrp_embedding(model,emb_name,n_words,feed,embedding_output_data,np.sum(lstm_lrp_x,axis=0),eps,delta,debug)
+
+        else:
+            #LRP through lstm layer
+            lstm_name = "lstm"        
+            feed = single_input_data
+            lstm_lrp_x,(lstm_lrp_h,lstm_lrp_g,lstm_lrp_c) = lrp_lstm(model,lstm_name,feed,T,d,lrp_fc,lstm_hidden,lstm_cell,lstm_actv1,lstm_actv2,eps,delta,debug)
 
-
 
-    return lrp_fc,lstm_lrp_x,(lstm_lrp_h,lstm_lrp_g,lstm_lrp_c)
+    return lrp_input,lrp_fc,lstm_lrp_x,(lstm_lrp_h,lstm_lrp_g,lstm_lrp_c)
 
 
 
-def lrp_full(model,input_filename,net_arch,net_arch_layers,test_data_json,fc_out_json,lstm_hidden_json,lstm_cell_json,eps,delta,save_dir,lstm_actv1=expit,lstm_actv2=np.tanh,topN=5,debug=False,predictions=None):
+def lrp_full(model,embedding_layer,n_words,input_filename,net_arch,net_arch_layers,test_data_json,fc_out_json,lstm_hidden_json,lstm_cell_json,ebd_json,eps,delta,save_dir,lstm_actv1=expit,lstm_actv2=np.tanh,topN=5,debug=False,predictions=None):
 
     LRP = collections.OrderedDict()
     totalLRP = collections.OrderedDict()
@@ -330,7 +360,7 @@ def lrp_full(model,input_filename,net_arch,net_arch_layers,test_data_json,fc_out
          kdata = np.array(list(data_test[k].values()))
          T = kdata.shape
 
-         lrp_fc,lstm_lrp_x,(lstm_lrp_h,lstm_lrp_g,lstm_lrp_c) = lrp_single_input(model,net_arch_layers,k,kdata,eps,delta,fc_out_json,lstm_hidden_json,lstm_cell_json,target_class=1,T=T,classes=2,lstm_actv1=expit,lstm_actv2=np.tanh,debug=debug)
+         lrp_input,lrp_fc,lstm_lrp_x,(lstm_lrp_h,lstm_lrp_g,lstm_lrp_c) = lrp_single_input(model,embedding_layer,n_words,net_arch_layers,k,kdata,eps,delta,fc_out_json,lstm_hidden_json,lstm_cell_json,ebd_json,target_class=1,T=T,classes=2,lstm_actv1=expit,lstm_actv2=np.tanh,debug=debug)
 
          lrp_neurons = get_topLRP_cells(lrp_fc,k,save_dir,topN)
          reviewLRP_data = get_PosNegNeurons_dict(i,predictions,lrp_neurons)

src/lrp_linear.py

@@ -28,6 +28,7 @@ def lrp_linear(hin, w, b, hout, Rout, bias_nb_units, eps, bias_factor, debug=Fal
     """
 
     sign_out = np.where(hout[na,:]>=0, 1., -1.) # shape (1, M)
+
     numer = np.zeros(w.shape)
     """
     for j in range(w.shape[1]):

src/parameter_persistence.py

@@ -13,6 +13,7 @@ def export_serial_lstm_data(model,layer_outputs,feed,input_files,data="lstm",sav
     lstm_states  = collections.OrderedDict()
     lstm_hidden  = collections.OrderedDict()  ## same as lstm_outputs, but latter includes the unncessary zeros at the end
     fc_outputs   = collections.OrderedDict()
+    ebd_outputs = collections.OrderedDict()
 
     with model.session.as_default():
 
@@ -54,10 +55,18 @@ def export_serial_lstm_data(model,layer_outputs,feed,input_files,data="lstm",sav
                data_out = layer_outputs[k].eval(feed_dict={'InputData/X:0':feed})
                for i in range(len(input_files)):
                    fc_outputs[input_files[i]] = data_out[i,:].tolist()
-
+
+           keys = [k for k in list(layer_outputs.keys()) if "embedding" in k]
+           for k in keys:
+               data_out = layer_outputs[k].eval(feed_dict={'InputData/X:0':feed})
+               for i in range(len(input_files)):
+                   ebd_outputs[input_files[i]] = data_out[i,:].tolist()
+
     if save_mode=="json":
         with open(save_dir+"model_internals_fc.json", 'w') as f:
             json.dump(fc_outputs, f)
+        with open(save_dir+"model_internals_ebd.json", 'w') as f:
+            json.dump(ebd_outputs, f)
         #with open(save_dir+"model_internals_lstm_outputs.json", 'w') as f:
         #    json.dump(lstm_outputs, f)
         with open(save_dir+"model_internals_lstm_hidden.json", 'w') as f:
@@ -67,6 +76,8 @@ def export_serial_lstm_data(model,layer_outputs,feed,input_files,data="lstm",sav
     elif save_mode=="pickle":
         with open(save_dir+"model_internals_fc.pickle", 'wb') as f:
             pickle.dump(fc_outputs, f)
+        with open(save_dir+"model_internals_ebd.pickle", 'wb') as f:
+            pickle.dump(ebd_outputs, f)
         #with open(save_dir+"model_internals_lstm_outputs.pickle", 'wb') as f:
         #    pickle.dump(lstm_outputs, f)
         with open(save_dir+"model_internals_lstm_hidden.pickle", 'wb') as f: