Solved bug in column name to column signature mapping for categorical…

… columns. Solved deprecation warnings.
alex-bogatu · Nov 18, 2021 · ce3874b · ce3874b
1 parent 8d74152
commit ce3874b
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 15 deletions.
diff --git a/d3l/indexing/feature_extraction/values/token_transformer.py b/d3l/indexing/feature_extraction/values/token_transformer.py
@@ -66,7 +66,7 @@ def transform(self, input_values: Iterable[str]) -> Set[str]:
         except ValueError:
             return set()
 
-        weight_map = dict(zip(vectorizer.get_feature_names(), vectorizer.idf_))
+        weight_map = dict(zip(vectorizer.get_feature_names_out(), vectorizer.idf_))
         tokenset = set()
         tokenizer = vectorizer.build_tokenizer()
         for value in input_values:

diff --git a/d3l/indexing/similarity_indexes/__init__.py b/d3l/indexing/similarity_indexes/__init__.py
@@ -148,8 +148,8 @@ def create_index(self) -> LSHIndex:
             lsh_index.add(input_id=str(table), input_set=table_signature)
             column_data = self.dataloader.get_columns(table_name=table)
 
-            column_signatures = [self.transformer.transform(c) for c in column_data]
-            for c, signature in zip(column_data, column_signatures):
+            column_signatures = [(c, self.transformer.transform(c)) for c in column_data]
+            for c, signature in column_signatures:
                 if len(signature) > 0:
                     lsh_index.add(input_id=str(table) + "." + str(c), input_set=signature)
 
@@ -249,11 +249,11 @@ def create_index(self) -> LSHIndex:
             table_data = self.dataloader.read_table(table_name=table)
 
             column_signatures = [
-                self.transformer.transform(table_data[c].tolist())
+                (c, self.transformer.transform(table_data[c].tolist()))
                 for c in table_data.columns
                 if not is_numeric(table_data[c]) and table_data[c].count() > 0
             ]
-            for c, signature in zip(table_data.columns.tolist(), column_signatures):
+            for c, signature in column_signatures:
                 if len(signature) > 0:
                     lsh_index.add(input_id=str(table) + "." + str(c), input_set=signature)
         return lsh_index
@@ -374,11 +374,11 @@ def create_index(self) -> LSHIndex:
             table_data = self.dataloader.read_table(table_name=table)
 
             column_signatures = [
-                self.transformer.transform(table_data[c].tolist())
+                (c, self.transformer.transform(table_data[c].tolist()))
                 for c in table_data.columns
                 if not is_numeric(table_data[c]) and table_data[c].count() > 0
             ]
-            for c, signature in zip(table_data.columns.tolist(), column_signatures):
+            for c, signature in column_signatures:
                 if len(signature) > 0:
                     lsh_index.add(input_id=str(table) + "." + str(c), input_set=signature)
 
@@ -519,11 +519,11 @@ def create_index(self) -> LSHIndex:
             table_data = self.dataloader.read_table(table_name=table)
 
             column_signatures = [
-                self.transformer.transform(table_data[c].tolist())
+                (c, self.transformer.transform(table_data[c].tolist()))
                 for c in table_data.columns
                 if not is_numeric(table_data[c]) and table_data[c].count() > 0
             ]
-            for c, signature in zip(table_data.columns.tolist(), column_signatures):
+            for c, signature in column_signatures:
                 if len(signature) > 0:
                     lsh_index.add(input_id=str(table) + "." + str(c), input_set=signature)
 
@@ -641,11 +641,11 @@ def create_index(self) -> LSHIndex:
             table_data = self.dataloader.read_table(table_name=table)
 
             column_signatures = [
-                self.transformer.transform(table_data[c].tolist())
+                (c, self.transformer.transform(table_data[c].tolist()))
                 for c in table_data.columns
                 if is_numeric(table_data[c]) and table_data[c].count() > 0
             ]
-            for c, signature in zip(table_data.columns.tolist(), column_signatures):
+            for c, signature in column_signatures:
                 if len(signature) > 0:
                     lsh_index.add(input_id=str(table) + "." + str(c), input_set=signature)
 

diff --git a/d3l/input_output/dataloaders/__init__.py b/d3l/input_output/dataloaders/__init__.py
@@ -491,15 +491,15 @@ def read_table(
                 usecols=table_columns,
                 chunksize=chunk_size,
                 low_memory=False,
-                error_bad_lines=False,
-                warn_bad_lines=False,
+                # error_bad_lines=False, # Deprecated in future versions
+                # warn_bad_lines=False, # Deprecated in future versions
                 **self.loading_kwargs
             )
         return pd.read_csv(
             file_path,
             chunksize=chunk_size,
             low_memory=False,
-            error_bad_lines=False,
-            warn_bad_lines=False,
+            # error_bad_lines=False, # Deprecated in future versions
+            # warn_bad_lines=False, # Deprecated in future versions
             **self.loading_kwargs
         )