Persephone refactor

egs/voxceleb/ssl.v1/README.md

@@ -137,6 +137,12 @@ run_xxx_xxxx.sh --config-file global_conf/other_config.sh
 | | | | | FT-2 | PLDA | 1.67 | 0.137 | 0.193 |
 | config_fbank80_stmn_fwseresnet34.v1.2.1.sh  | FW-SE ResNet34 | Cos+AHC+PLDA+AHC | Cos+AHC | FT-2 | Cosine | 1.49 | 0.101 | 0.161 |
 | | | | | FT-2 | PLDA | 1.53 | 0.109 | 0.168|
+| config_fbank80_stmn_fwseresnet34.v1.2.2.sh  | FW-SE ResNet34 / 0.1 x Cos Reg. | Cos+AHC+PLDA+AHC | Cos+AHC | DINO | Cosine | 3.96 | 0.232 | 0.358 |
+| | | | | | PLDA | 4.04 | 0.185 | 0.291 |
+| | | | | FT-1 | Cosine | 2.03 | 0.125 | 0.203 |
+| | | | | FT-1 | PLDA | 2.44 | 0.149 | 0.231 |
+| | | | | FT-2 | Cosine | 
+| | | | | FT-2 | PLDA | 
 
 
 ### VoxCeleb 1 Entire-Clean trial list
@@ -161,6 +167,12 @@ run_xxx_xxxx.sh --config-file global_conf/other_config.sh
 | | | | | FT-2 | PLDA | 1.77 | 0.121 | 0.208 |
 | config_fbank80_stmn_fwseresnet34.v1.2.1.sh  | FW-SE ResNet34 | Cos+AHC+PLDA+AHC | Cos+AHC | FT-2 | Cosine | 1.83 | 0.106 | 0.170 |
 | | | | | FT-2 | PLDA | 1.68 | 0.109 | 0.188 |
+| config_fbank80_stmn_fwseresnet34.v1.2.2.sh  | FW-SE ResNet34 / 0.1 x Cos Reg. | Cos+AHC+PLDA+AHC | Cos+AHC | DINO | Cosine | 4.31 | 0.250 | 0.387 |
+| | | | | | PLDA | 4.32 | 0.166 | 0.263 |
+| | | | | FT-1 | Cosine | 2.61 | 0.138 | 0.210 | 
+| | | | | FT-1 | PLDA | 2.72 | 0.1366 | 0.216 |
+| | | | | FT-2 | Cosine | 
+| | | | | FT-2 | PLDA | 
 
 
 ### VoxCeleb 1 Hard-Clean trial list
@@ -185,4 +197,10 @@ run_xxx_xxxx.sh --config-file global_conf/other_config.sh
 | | | | | FT-2 | PLDA | 3.51 | 0.219 | 0.351 |
 | config_fbank80_stmn_fwseresnet34.v1.2.1.sh  | FW-SE ResNet34 | Cos+AHC+PLDA+AHC | Cos+AHC | FT-2 | Cosine | 3.11 | 0.172 | 0.270 |
 | | | | | FT-2 | PLDA | 3.15 | 0.186 | 0.294 |
+| config_fbank80_stmn_fwseresnet34.v1.2.2.sh  | FW-SE ResNet34 / 0.1 x Cos Reg. | Cos+AHC+PLDA+AHC | Cos+AHC | DINO | Cosine | 7.41 | 0.377 | 0.526 |
+| | | | | | PLDA | 5.95 | 0.269 | 0.438 |
+| | | | | FT-1 | Cosine | 4.38 | 0.222 | 0.337 |
+| | | | | FT-1 | PLDA | 4.68 | 0.237 | 0.375 |
+| | | | | FT-2 | Cosine | 
+| | | | | FT-2 | PLDA | 
 

egs/voxceleb/ssl.v1/conf/train_fwseresnet34_dino_v1.2.2.yaml

@@ -80,7 +80,7 @@ trainer:
     decay_rate: 0.5
     decay_steps: 60000
     hold_steps: 15000
-    min_lr: 1.0e-05
+    min_lr: 1.0e-04
     warmup_steps: 15000
     update_lr_on_opt_step: true
   teacher_optim:

egs/voxceleb/ssl.v1/global_conf/config_fbank80_stmn_ecapatdnn512x3.v1.2.sh

@@ -43,7 +43,7 @@ nnet_ft_s1_2=$nnet_ft_s1_2_dir/model_ep0070.pth
 # clustering of ft embeddings from stage 1.2
 cluster_ft_s1_method=cos_ahc_plda_ahc
 cluster_ft_s1_cfg=conf/cluster_ecapatdnn512x3_v1.2_ft1_cos_ahc_plda_ahc.yaml
-cluster_ft_s1_name=${cluster_method}
+cluster_ft_s1_name=${cluster_ft_s1_method}
 cluster_ft_s1_dir=exp/clustering/$nnet_ft_s1_2_name/$cluster_ft_s1_name
 
 
@@ -62,7 +62,7 @@ nnet_ft_s2_2=$nnet_ft_s2_2_dir/model_ep0070.pth
 # clustering of ft embeddings from stage 1.2
 cluster_ft_s2_method=cos_ahc_plda_ahc
 cluster_ft_s2_cfg=conf/cluster_ecapatdnn512x3_v1.2_ft1_cos_ahc_plda_ahc.yaml
-cluster_ft_s2_name=${cluster_method}
+cluster_ft_s2_name=${cluster_ft_s2_method}
 cluster_ft_s2_dir=exp/clustering/$nnet_ft_s2_2_name/$cluster_ft_s2_name
 
 

egs/voxceleb/ssl.v1/global_conf/config_fbank80_stmn_fwseresnet34.v1.2.1.sh

@@ -43,7 +43,7 @@ nnet_ft_s1_2=$nnet_ft_s1_2_dir/model_ep0070.pth
 # clustering of ft embeddings from stage 1.2
 cluster_ft_s1_method=cos_ahc
 cluster_ft_s1_cfg=conf/cluster_lresnet34_v1.2_ft1_cos_ahc.yaml
-cluster_ft_s1_name=${cluster_method_ft_s1_method}
+cluster_ft_s1_name=${cluster_ft_s1_method}
 cluster_ft_s1_dir=exp/clustering/$nnet_ft_s1_2_name/$cluster_ft_s1_name
 
 # finetuning stage 2.1
@@ -61,6 +61,6 @@ nnet_ft_s2_2=$nnet_ft_s2_2_dir/model_ep0070.pth
 # clustering of ft embeddings from stage 2.2
 cluster_ft_s2_method=cos_ahc
 cluster_ft_s2_cfg=conf/cluster_lresnet34_v1.2_ft1_cos_ahc.yaml
-cluster_ft_s2_name=${cluster_method_ft_s2_method}
+cluster_ft_s2_name=${cluster_ft_s2_method}
 cluster_ft_s2_dir=exp/clustering/$nnet_ft_s2_2_name/$cluster_ft_s2_name
 

egs/voxceleb/ssl.v1/global_conf/config_fbank80_stmn_fwseresnet34.v1.2.2.sh

@@ -17,21 +17,7 @@ nnet_name=${feat_type}_fwseresnet34_dino.v1.2.2
 nnet_s1_base_cfg=conf/train_fwseresnet34_dino_v1.2.2.yaml
 nnet_s1_name=$nnet_name.s1
 nnet_s1_dir=exp/xvector_nnets/$nnet_s1_name
-nnet_s1=$nnet_s1_dir/teacher_model_ep0034.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0038.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0043.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0044.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0046.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0049.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0054.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0058.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0064.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0067.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0071.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0077.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0083.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0088.pth
-nnet_s1=$nnet_s1_dir/teacher_model_ep0094.pth
+nnet_s1=$nnet_s1_dir/teacher_model_ep0100.pth
 
 # clustering of dino embeddings
 cluster_method=cos_ahc_plda_ahc
@@ -43,38 +29,38 @@ cluster_dir=exp/clustering/$nnet_s1_name/$cluster_name
 plda_cfg=conf/plda.yaml
 
 # finetuning stage 1.1
-nnet_ft_s1_1_base_cfg=conf/train_lresnet34_xvec_stage1.1_v1.2.yaml
+nnet_ft_s1_1_base_cfg=conf/train_fwseresnet34_xvec_stage1.1_v1.2.2.yaml
 nnet_ft_s1_1_name=$nnet_name.s1.ft.s1.1
 nnet_ft_s1_1_dir=exp/xvector_nnets/$nnet_ft_s1_1_name
-nnet_ft_s1_1=$nnet_ft_s1_1_dir/model_ep0030.pth
+nnet_ft_s1_1=$nnet_ft_s1_1_dir/model_ep0025.pth
 
 # finetuning stage 1.2
-nnet_ft_s1_2_base_cfg=conf/train_lresnet34_xvec_stage1.2_v1.2.yaml
+nnet_ft_s1_2_base_cfg=conf/train_fwseresnet34_xvec_stage1.2_v1.2.2.yaml
 nnet_ft_s1_2_name=$nnet_name.s1.ft.s1.2
 nnet_ft_s1_2_dir=exp/xvector_nnets/$nnet_ft_s1_2_name
 nnet_ft_s1_2=$nnet_ft_s1_2_dir/model_ep0070.pth
 
 # clustering of ft embeddings from stage 1.2
 cluster_ft_s1_method=cos_ahc
 cluster_ft_s1_cfg=conf/cluster_lresnet34_v1.2_ft1_cos_ahc.yaml
-cluster_ft_s1_name=${cluster_method_ft_s1_method}
+cluster_ft_s1_name=${cluster_ft_s1_method}
 cluster_ft_s1_dir=exp/clustering/$nnet_ft_s1_2_name/$cluster_ft_s1_name
 
 # finetuning stage 2.1
-nnet_ft_s2_1_base_cfg=conf/train_lresnet34_xvec_stage1.1_v1.2.yaml
+nnet_ft_s2_1_base_cfg=conf/train_fwseresnet34_xvec_stage1.1_v1.2.2.yaml
 nnet_ft_s2_1_name=$nnet_name.s1.ft.s2.1
 nnet_ft_s2_1_dir=exp/xvector_nnets/$nnet_ft_s2_1_name
 nnet_ft_s2_1=$nnet_ft_s2_1_dir/model_ep0030.pth
 
 # finetuning stage 2.2
-nnet_ft_s2_2_base_cfg=conf/train_lresnet34_xvec_stage1.2_v1.2.yaml
+nnet_ft_s2_2_base_cfg=conf/train_fwseresnet34_xvec_stage1.2_v1.2.2.yaml
 nnet_ft_s2_2_name=$nnet_name.s1.ft.s2.2
 nnet_ft_s2_2_dir=exp/xvector_nnets/$nnet_ft_s2_2_name
 nnet_ft_s2_2=$nnet_ft_s2_2_dir/model_ep0070.pth
 
 # clustering of ft embeddings from stage 2.2
 cluster_ft_s2_method=cos_ahc
 cluster_ft_s2_cfg=conf/cluster_lresnet34_v1.2_ft1_cos_ahc.yaml
-cluster_ft_s2_name=${cluster_method_ft_s2_method}
+cluster_ft_s2_name=${cluster_ft_s2_method}
 cluster_ft_s2_dir=exp/clustering/$nnet_ft_s2_2_name/$cluster_ft_s2_name
 

egs/voxceleb/ssl.v1/global_conf/config_fbank80_stmn_fwseresnet34.v1.2.sh

@@ -43,7 +43,7 @@ nnet_ft_s1_2=$nnet_ft_s1_2_dir/model_ep0070.pth
 # clustering of ft embeddings from stage 1.2
 cluster_ft_s1_method=cos_ahc_plda_ahc
 cluster_ft_s1_cfg=conf/cluster_lresnet34_v1.2_ft1_cos_ahc_plda_ahc.yaml
-cluster_ft_s1_name=${cluster_method}
+cluster_ft_s1_name=${cluster_fs_s1_method}
 cluster_ft_s1_dir=exp/clustering/$nnet_ft_s1_2_name/$cluster_ft_s1_name
 
 # finetuning stage 2.1
@@ -61,6 +61,6 @@ nnet_ft_s2_2=$nnet_ft_s2_2_dir/model_ep0070.pth
 # clustering of ft embeddings from stage 2.2
 cluster_ft_s2_method=cos_ahc_plda_ahc
 cluster_ft_s2_cfg=conf/cluster_lresnet34_v1.2_ft1_cos_ahc_plda_ahc.yaml
-cluster_ft_s2_name=${cluster_method}
+cluster_ft_s2_name=${cluster_ft_s2_method}
 cluster_ft_s2_dir=exp/clustering/$nnet_ft_s2_2_name/$cluster_ft_s2_name
 

egs/voxceleb/ssl.v1/global_conf/config_fbank80_stmn_lresnet34.v1.2.sh

@@ -43,7 +43,7 @@ nnet_ft_s1_2=$nnet_ft_s1_2_dir/model_ep0070.pth
 # clustering of ft embeddings from stage 1.2
 cluster_ft_s1_method=cos_ahc_plda_ahc
 cluster_ft_s1_cfg=conf/cluster_lresnet34_v1.2_ft1_cos_ahc_plda_ahc.yaml
-cluster_ft_s1_name=${cluster_method}
+cluster_ft_s1_name=${cluster_ft_s1_method}
 cluster_ft_s1_dir=exp/clustering/$nnet_ft_s1_2_name/$cluster_ft_s1_name
 
 # finetuning stage 2.1
@@ -61,5 +61,5 @@ nnet_ft_s2_2=$nnet_ft_s2_2_dir/model_ep0070.pth
 # clustering of ft embeddings from stage 2.2
 cluster_ft_s2_method=cos_ahc_plda_ahc
 cluster_ft_s2_cfg=conf/cluster_lresnet34_v1.2_ft1_cos_ahc_plda_ahc.yaml
-cluster_ft_s2_name=${cluster_method}
+cluster_ft_s2_name=${cluster_ft_s2_method}
 cluster_ft_s2_dir=exp/clustering/$nnet_ft_s2_2_name/$cluster_ft_s2_name

hyperion/np/feats/feature_windows.py

@@ -6,7 +6,7 @@
 import logging
 
 import numpy as np
-from scipy.signal import blackman, hamming, hann
+from scipy.signal.windows import blackman, hamming, hann
 
 from ...hyp_defs import float_cpu
 

hyperion/torch/data/audio_dataset.py

@@ -183,6 +183,9 @@ def _load_class_infos(self, class_names, class_files, is_val):
             assert (
                 name in self.seg_set
             ), f"class_name {name} not present in the segment set"
+            self.seg_set.convert_col_to_str(
+                name
+            )  # make sure that class ids are strings
             if self.rank == 0:
                 logging.info("loading class-info file %s", file)
             table = ClassInfo.load(file)

hyperion/torch/trainers/xvector_adv_trainer.py

@@ -154,7 +154,7 @@ def train_epoch(self, data_loader):
 
             batch_metrics["loss"] = loss.item() * self.grad_acc_steps
             for k, metric in self.metrics.items():
-                batch_metrics[k] = metric(output, target)
+                batch_metrics[k] = metric(output.logits, target)
 
             metric_acc.update(batch_metrics, batch_size)
             logs = metric_acc.metrics
@@ -198,7 +198,7 @@ def validation_epoch(self, data_loader, swa_update_bn=False):
 
             batch_metrics["loss"] = loss.item()
             for k, metric in self.metrics.items():
-                batch_metrics[k] = metric(output, target)
+                batch_metrics[k] = metric(output.logits, target)
 
             metric_acc.update(batch_metrics, batch_size)
 

hyperion/torch/trainers/xvector_adv_trainer_from_wav.py

@@ -159,7 +159,7 @@ def train_epoch(self, data_loader):
 
             batch_metrics["loss"] = loss.item() * self.grad_acc_steps
             for k, metric in self.metrics.items():
-                batch_metrics[k] = metric(output, target)
+                batch_metrics[k] = metric(output.logits, target)
 
             metric_acc.update(batch_metrics, batch_size)
             logs = metric_acc.metrics
@@ -203,7 +203,7 @@ def validation_epoch(self, data_loader, swa_update_bn=False):
 
             batch_metrics["loss"] = loss.item()
             for k, metric in self.metrics.items():
-                batch_metrics[k] = metric(output, target)
+                batch_metrics[k] = metric(output.logits, target)
 
             metric_acc.update(batch_metrics, batch_size)
 

hyperion/torch/trainers/xvector_trainer.py

@@ -137,7 +137,7 @@ def train_epoch(self, data_loader):
 
             batch_metrics["loss"] = loss_acc * self.grad_acc_steps
             for k, metric in self.metrics.items():
-                batch_metrics[k] = metric(output, target)
+                batch_metrics[k] = metric(output.logits, target)
 
             metric_acc.update(batch_metrics, batch_size)
             logs = metric_acc.metrics
@@ -185,7 +185,7 @@ def validation_epoch(self, data_loader, swa_update_bn=False):
 
                 batch_metrics["loss"] = loss_acc
                 for k, metric in self.metrics.items():
-                    batch_metrics[k] = metric(output, target)
+                    batch_metrics[k] = metric(output.logits, target)
 
                 metric_acc.update(batch_metrics, batch_size)
 

hyperion/torch/trainers/xvector_trainer_deep_feat_reg_from_wav.py

@@ -193,7 +193,7 @@ def train_epoch(self, data_loader):
                 self.save_checkpoint(partial=True)
 
             for k, metric in self.metrics.items():
-                batch_metrics[k] = metric(output, target)
+                batch_metrics[k] = metric(output.logits, target)
 
             metric_acc.update(batch_metrics, batch_size)
             logs = metric_acc.metrics
@@ -235,7 +235,7 @@ def validation_epoch(self, data_loader, swa_update_bn=False):
 
                 batch_metrics["loss"] = loss.item()
                 for k, metric in self.metrics.items():
-                    batch_metrics[k] = metric(output, target)
+                    batch_metrics[k] = metric(output.logits, target)
 
                 metric_acc.update(batch_metrics, batch_size)
 

hyperion/torch/trainers/xvector_trainer_from_wav.py

@@ -133,7 +133,7 @@ def train_epoch(self, data_loader):
 
             batch_metrics["loss"] = loss.item() * self.grad_acc_steps
             for k, metric in self.metrics.items():
-                batch_metrics[k] = metric(output, target)
+                batch_metrics[k] = metric(output.logits, target)
 
             metric_acc.update(batch_metrics, batch_size)
             logs = metric_acc.metrics
@@ -177,7 +177,7 @@ def validation_epoch(self, data_loader, swa_update_bn=False):
 
                 batch_metrics["loss"] = loss.mean().item()
                 for k, metric in self.metrics.items():
-                    batch_metrics[k] = metric(output, target)
+                    batch_metrics[k] = metric(output.logits, target)
 
                 metric_acc.update(batch_metrics, batch_size)
 

hyperion/utils/hyp_dataset.py

@@ -95,6 +95,16 @@ def __init__(
         self.sparse_trials = sparse_trials
         self.table_sep = table_sep
         self._files_to_delete = []
+        self.fix_segments_dtypes()
+
+    def fix_segments_dtypes(self):
+        if self._segments is not None:
+            self._fix_segments_dtypes(self._segments)
+
+    def _fix_segments_dtypes(self, segments):
+        # ids in class_infos should be strings in segment set columns
+        for k in self.classes_keys():
+            segments.convert_col_to_str(k)
 
     def get_dataset_files(self):
         file_paths = []
@@ -149,6 +159,7 @@ def segments(self, keep_loaded: bool = True):
         if self._segments is None:
             assert self._segments_path is not None
             segments = SegmentSet.load(self._segments_path, sep=self.table_sep)
+            self._fix_segments_dtypes(segments)
             if keep_loaded:
                 self._segments = segments
             return segments

hyperion/utils/info_table.py

@@ -33,7 +33,11 @@ def __init__(self, df):
 
     def fix_dtypes(self):
         if infer_dtype(self.df.id) != "string":
-            self.df[:, "id"] = self.df["id"].apply(str)
+            self.df.loc[:, "id"] = self.df["id"].apply(str)
+
+    def convert_col_to_str(self, column):
+        if infer_dtype(self.df[column]) != "string":
+            self.df.loc[:, column] = self.df[column].apply(str)
 
     def copy(self):
         """Makes a copy of the object."""