Successfully trained abuse and intent models

Author: simonsben

execution/analysis/embeddings/collect_intent_verbs.py

@@ -12,17 +12,17 @@
 destination_dir = base_dir / 'embeddings'
 
 frame_info_path = data_dir / 'intent_frame.csv'
-english_mask = data_dir / 'english_mask.csv'
+# english_mask = data_dir / 'english_mask.csv'
 
 desire_index = 1
 action_index = 2
 
-check_existence([frame_info_path, model_path, english_mask])
+check_existence([frame_info_path, model_path])
 make_dir(destination_dir)
 print('Config complete.')
 
-english_mask = load_vector(english_mask).astype(bool)
-intent_frames = read_csv(frame_info_path, header=None, keep_default_na=False).values[english_mask]
+# english_mask = load_vector(english_mask).astype(bool)
+intent_frames = read_csv(frame_info_path, header=None, keep_default_na=False).values
 print('Loaded data with shape', intent_frames.shape)
 
 desire_verbs = get_verbs(intent_frames, desire_index)

execution/prediction/abusive_intent.py

@@ -1,10 +1,10 @@
 from utilities.data_management import make_path, check_existence, open_w_pandas, get_model_path, vector_to_file, \
-    make_dir
+    make_dir, get_embedding_path
 from model.networks import predict_abusive_intent
 from config import dataset
 
 
-embedding_path = make_path('data/models/') / dataset / 'derived' / (dataset + '.bin')
+embedding_path = get_embedding_path()
 processed_base = make_path('data/processed_data') / dataset / 'analysis'
 context_path = processed_base / 'intent' / 'contexts.csv'
 predictions_base = processed_base / 'intent_abuse'

execution/training/abuse.py

@@ -1,21 +1,24 @@
 from model.networks import generate_abuse_network
 from utilities.data_management import make_dir, make_path, open_w_pandas, check_existence, \
-    get_model_path, load_vector, vector_to_file, split_sets
+    get_model_path, load_vector, vector_to_file, split_sets, get_embedding_path
 from fasttext import load_model
 from model.layers.realtime_embedding import RealtimeEmbedding
 from keras.callbacks import EarlyStopping
 from config import dataset, max_tokens, training_verbosity, batch_size
+from time import time
 
 
 # Define paths
 abuse_weights_path = get_model_path('abuse')
-embedding_path = make_path('data/models/') / dataset / 'derived' / (dataset + '.bin')
+embedding_path = get_embedding_path()
 base_path = make_path('data/processed_data/') / dataset / 'analysis'
 data_path = make_path('data/prepared_data/abusive_data.csv')
+dest_dir = base_path / 'abuse'
 
 # Check for files and make directories
 check_existence([embedding_path, data_path])
 make_dir(abuse_weights_path.parent)
+make_dir(dest_dir)
 print('Config complete.')
 
 # Load embeddings and contexts
@@ -41,12 +44,24 @@
 training_steps = int(len(training_data) / batch_size) + 1
 validation_steps = int(len(testing_data) / batch_size) + 1
 
-stopping_conditions = EarlyStopping(monitor='val_loss', patience=2, verbose=1, restore_best_weights=True)
-model.fit_generator(training, epochs=50, verbose=training_verbosity, callbacks=[stopping_conditions],
-                    validation_data=testing, shuffle=True)
+start = time()
 
+stopping_conditions = EarlyStopping(monitor='val_loss', patience=3, verbose=1, restore_best_weights=True)
+history = model.fit_generator(training, epochs=50, verbose=training_verbosity, callbacks=[stopping_conditions],
+                    validation_data=testing, shuffle=True).history
+
+training_time = time() - start
+print('Completed training in', training_time, 's')
+print('Training history', history)
 
 evaluated_accuracy = model.evaluate_generator(testing, verbose=training_verbosity, steps=validation_steps)
 print('Model validation accuracy', evaluated_accuracy)
 
 model.save_weights(str(abuse_weights_path))
+print('Completed training and saving abuse model.')
+
+vector_to_file(training.data_source, dest_dir / 'training_data.csv')
+vector_to_file(training.labels, dest_dir / 'training_labels.csv')
+
+vector_to_file(testing.data_source, dest_dir / 'testing_data.csv')
+vector_to_file(testing.labels, dest_dir / 'testing_labels.csv')

execution/training/intent.py

@@ -30,7 +30,6 @@
 # Load embeddings and contexts
 embedding_model = load_model(str(embedding_path))
 
-# english_mask = load_vector(english_mask_path).astype(bool)
 raw_contexts = open_w_pandas(context_path)['contexts'].values
 initial_labels = load_vector(initial_label_path)
 document_matrix = load_npz(document_matrix_path)

model/layers/realtime_embedding.py

@@ -33,7 +33,7 @@ def __init__(self, embedding_model, data_source, labels=None, labels_in_progress
         self.working_initial_labels = self.original_initial_labels
         self.is_training = False
 
-        self.concrete_weight = 1.5
+        self.concrete_weight = 2
         self.midpoint = 0.5
         self.uniform_weights = uniform_weights
         self.data_length = ceil(len(self.working_data_source) / batch_size)

utilities/data_management/io.py

@@ -185,6 +185,7 @@ def output_abusive_intent(index_set, predictions, contexts, filename=None):
     'O': '%s',
     'U': '%s',
     'i': '%d',
+    'b': '%d',
     'f': '%.6f'
 }