including comments from 01.28.

Author: Louquinze

autosklearn/pipeline/components/data_preprocessing/text_encoding/bag_of_word_encoding.py

@@ -35,6 +35,7 @@ def fit(self, X: PIPELINE_DATA_DTYPE, y: Optional[PIPELINE_DATA_DTYPE] = None
             ) -> 'BagOfWordEncoder':
 
         if isinstance(X, pd.DataFrame):
+            X.fillna("", inplace=True)
             # define a CountVectorizer for every feature (implicitly defined by order of columns,
             # maybe change the list
             # to a dictionary with features as keys)
@@ -47,7 +48,7 @@ def fit(self, X: PIPELINE_DATA_DTYPE, y: Optional[PIPELINE_DATA_DTYPE] = None
             else:
                 raise KeyError()
 
-            all_text = itertools.chain.from_iterable(X[col].dropna() for col in X.columns)
+            all_text = itertools.chain.from_iterable(X[col] for col in X.columns)
             self.preprocessor = self.preprocessor.fit(all_text)
 
         else:

autosklearn/pipeline/components/data_preprocessing/text_encoding/bag_of_word_encoding_distinct.py

@@ -34,14 +34,14 @@ def fit(self, X: PIPELINE_DATA_DTYPE, y: Optional[PIPELINE_DATA_DTYPE] = None
             ) -> 'BagOfWordEncoder':
 
         if isinstance(X, pd.DataFrame):
+            X.fillna("", inplace=True)
             if self.min_df_choice == "min_df_absolute":
 
                 self.preprocessor = {}
 
                 for feature in X.columns:
                     vectorizer = CountVectorizer(min_df=self.min_df_absolute,
-                                                 ngram_range=(1, self.ngram_range)).fit(
-                        X[feature].dropna())
+                                                 ngram_range=(1, self.ngram_range)).fit(X[feature])
                     self.preprocessor[feature] = vectorizer
 
             elif self.min_df_choice == "min_df_relative":
@@ -50,8 +50,7 @@ def fit(self, X: PIPELINE_DATA_DTYPE, y: Optional[PIPELINE_DATA_DTYPE] = None
 
                 for feature in X.columns:
                     vectorizer = CountVectorizer(min_df=self.min_df_relative,
-                                                 ngram_range=(1, self.ngram_range)).fit(
-                        X[feature].dropna())
+                                                 ngram_range=(1, self.ngram_range)).fit(X[feature])
                     self.preprocessor[feature] = vectorizer
             else:
                 raise KeyError()

autosklearn/pipeline/components/data_preprocessing/text_encoding/tfidf_encoding.py

@@ -36,6 +36,7 @@ def fit(self, X: PIPELINE_DATA_DTYPE, y: Optional[PIPELINE_DATA_DTYPE] = None
             ) -> 'TfidfEncoder':
 
         if isinstance(X, pd.DataFrame):
+            X.fillna("", inplace=True)
             # define a CountVectorizer for every feature (implicitly defined by order of columns,
             # maybe change the list
             # to a dictionary with features as keys)
@@ -50,7 +51,7 @@ def fit(self, X: PIPELINE_DATA_DTYPE, y: Optional[PIPELINE_DATA_DTYPE] = None
             else:
                 raise KeyError()
 
-            all_text = itertools.chain.from_iterable(X[col].dropna() for col in X.columns)
+            all_text = itertools.chain.from_iterable(X[col] for col in X.columns)
             self.preprocessor = self.preprocessor.fit(all_text)
 
         else: