update

Author: bob7783

nlp_class/spam2.py

@@ -35,15 +35,19 @@
 df['b_labels'] = df['labels'].map({'ham': 0, 'spam': 1})
 Y = df['b_labels'].values
 
+# split up the data
+df_train, df_test, Ytrain, Ytest = train_test_split(df['data'], Y, test_size=0.33)
+
 # try multiple ways of calculating features
-# tfidf = TfidfVectorizer(decode_error='ignore')
-# X = tfidf.fit_transform(df['data'])
+tfidf = TfidfVectorizer(decode_error='ignore')
+Xtrain = tfidf.fit_transform(df_train)
+Xtest = tfidf.transform(df_test)
+
+# count_vectorizer = CountVectorizer(decode_error='ignore')
+# Xtrain = count_vectorizer.fit_transform(df_train)
+# Xtest = count_vectorizer.transform(df_test)
 
-count_vectorizer = CountVectorizer(decode_error='ignore')
-X = count_vectorizer.fit_transform(df['data'])
 
-# split up the data
-Xtrain, Xtest, Ytrain, Ytest = train_test_split(X, Y, test_size=0.33)
 
 # create the model, train it, print scores
 model = MultinomialNB()