Dnnc with no finetuning

deeppavlov/configs/classifiers/dnnc_infer.json

@@ -0,0 +1,99 @@
+{
+  "dataset_reader": {
+    "class_name": "basic_classification_reader",
+    "format": "json",
+    "orient": "split",
+    "x": "text",
+    "y": "category",
+    "data_path": "{DOWNLOADS_PATH}",
+    "train": "train.json",
+    "valid": "dev.json",
+    "test": "test.json"
+  },
+  "dataset_iterator": {
+    "class_name": "basic_classification_iterator",
+    "seed": 42,
+    "shuffle": true
+  },
+  "chainer": {
+    "in": ["input"],
+    "in_y": ["y_true"],
+    "pipe": [
+      {
+        "class_name": "dnnc_input_preprocessor",
+        "in": ["input"],
+        "out": ["x", "x_support", "x_populated", "y_support"],
+        "support_dataset_path": "{SUPPORT_DATA_PATH}/support_dataset.json",
+        "bidirectional": true
+      },
+      {
+        "class_name": "torch_transformers_preprocessor",
+        "in": ["x_populated", "x_support"],
+        "out": ["bert_features"],
+        "vocab_file": "{BASE_MODEL}",
+        "do_lower_case": true,
+        "max_seq_length": 128
+      },
+      {
+        "class_name": "torch_transformers_classifier",
+        "main": true,
+        "in": ["bert_features"],
+        "out": ["simmilarity_scores"],
+        "n_classes": 2,
+        "return_probas": true,
+        "pretrained_bert": "{BASE_MODEL}",
+        "save_path": "{MODEL_PATH}/model",
+        "load_path": "{MODEL_PATH}/model",
+        "is_binary": "{BINARY_CLASSIFICATION}",
+        "optimizer_parameters": {"lr": 2e-05}
+      },
+      {
+        "class_name": "dnnc_proba2labels",
+        "is_binary": "{BINARY_CLASSIFICATION}",
+        "in": ["simmilarity_scores", "x", "x_populated", "x_support", "y_support"],
+        "out": ["y_pred"],
+        "multilabel": false,
+        "confidence_threshold": 0.0,
+        "pooling": "max"
+      }
+    ],
+    "out": ["y_pred"]
+  },
+  "train": {
+    "batch_size": 1,
+    "metrics": [
+      {
+        "name": "accuracy_oos",
+        "inputs": ["y_true", "y_pred"],
+        "exclude_oos": true
+      },
+      {
+        "name": "oos_scores",
+        "inputs": ["y_true", "y_pred"]
+      }
+    ],
+    "show_examples": false,
+    "evaluation_targets": ["test"],
+    "class_name": "torch_trainer"
+  },
+  "metadata": {
+    "variables": {
+      "ROOT_PATH": "~/.deeppavlov",
+      "DOWNLOADS_PATH": "{ROOT_PATH}/downloads/clinc150",
+      "SUPPORT_DATA_PATH": "{ROOT_PATH}/preprocessed_datasets",
+      "MODEL_PATH": "{ROOT_PATH}/models/fewshot/roberta_nli_mrpc_1_10",
+      "BINARY_CLASSIFICATION": true,
+      "BASE_MODEL": "roberta-base"
+    },
+    "download": [
+      {
+        "url": "http://files.deeppavlov.ai/v1/classifiers/fewshot/roberta_nli_mrpc_1_10.tar.gz",
+        "subdir": "{MODEL_PATH}"
+      },
+      {
+        "url": "https://files.deeppavlov.ai/datasets/clinc150.tar.gz",
+        "subdir": "{DOWNLOADS_PATH}"
+      }
+    ]
+  }
+}

deeppavlov/core/common/metrics_registry.json

@@ -1,6 +1,6 @@
 {
   "acc": "deeppavlov.metrics.accuracy:round_accuracy",
-  "accuracy": "deeppavlov.metrics.accuracy:accuracy",
+  "accuracy_oos": "deeppavlov.metrics.accuracy:accuracy_oos",
   "average__ner_f1__f1_macro__f1": "deeppavlov.metrics.fmeasure:ner_f1__f1_macro__f1",
   "average__roc_auc__roc_auc__ner_f1": "deeppavlov.metrics.fmeasure:roc_auc__roc_auc__ner_f1",
   "bleu": "deeppavlov.metrics.bleu:bleu",
@@ -19,6 +19,7 @@
   "multitask_token_accuracy": "deeppavlov.metrics.accuracy:multitask_token_accuracy",
   "ner_f1": "deeppavlov.metrics.fmeasure:ner_f1",
   "ner_token_f1": "deeppavlov.metrics.fmeasure:ner_token_f1",
+  "oos_scores": "deeppavlov.metrics.fmeasure:oos_scores",
   "pearson_correlation": "deeppavlov.metrics.correlation:pearson_correlation",
   "per_item_bleu": "deeppavlov.metrics.bleu:per_item_bleu",
   "per_item_dialog_accuracy": "deeppavlov.metrics.accuracy:per_item_dialog_accuracy",

deeppavlov/core/common/registry.json

@@ -12,9 +12,12 @@
   "dirty_comments_preprocessor": "deeppavlov.models.preprocessors.dirty_comments_preprocessor:DirtyCommentsPreprocessor",
   "docred_reader": "deeppavlov.dataset_readers.docred_reader:DocREDDatasetReader",
   "document_chunker": "deeppavlov.models.preprocessors.odqa_preprocessors:DocumentChunker",
+  "dnnc_input_preprocessor": "deeppavlov.models.preprocessors.dnnc_preprocessor:InputPreprocessor",
+  "dnnc_proba2labels": "deeppavlov.models.classifiers.dnnc_proba2labels:Proba2Labels",
   "entity_detection_parser": "deeppavlov.models.entity_extraction.entity_detection_parser:EntityDetectionParser",
   "entity_linker": "deeppavlov.models.entity_extraction.entity_linking:EntityLinker",
   "faq_reader": "deeppavlov.dataset_readers.faq_reader:FaqDatasetReader",
+  "few_shot_iterator": "deeppavlov.dataset_iterators.few_shot_iterator:FewShotIterator",
   "fasttext": "deeppavlov.models.embedders.fasttext_embedder:FasttextEmbedder",
   "fit_trainer": "deeppavlov.core.trainers.fit_trainer:FitTrainer",
   "hashing_tfidf_vectorizer": "deeppavlov.models.vectorizers.hashing_tfidf_vectorizer:HashingTfIdfVectorizer",

deeppavlov/dataset_iterators/few_shot_iterator.py

@@ -0,0 +1,147 @@
+# Copyright 2017 Neural Networks and Deep Learning lab, MIPT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+from logging import getLogger
+from pathlib import Path
+from random import Random
+from typing import Dict, Any, List, Tuple, Optional
+
+from deeppavlov.core.common.registry import register
+from deeppavlov.core.data.data_learning_iterator import DataLearningIterator
+
+ENTAILMENT = 'entailment'
+NON_ENTAILMENT = 'non_entailment'
+
+SUPPORT_DATASET_PATH = "~/.deeppavlov/preprocessed_datasets/support_dataset.json"
+
+log = getLogger(__name__)
+
+
+@register('few_shot_iterator')
+class FewShotIterator(DataLearningIterator):
+    def __init__(self,
+                 data: Dict[str, List[Tuple[Any, Any]]],
+                 seed: int = None,
+                 shuffle: bool = True,
+                 shot: Optional[int] = None,
+                 shot_test: Optional[int] = None,
+                 return_nli_format: bool = False,
+                 *args, **kwargs) -> None:
+        self.shuffle = shuffle
+        self.random = Random(seed)
+
+        self.train = self.delete_oos(data.get('train', []))
+        self.valid = self.delete_oos(data.get('valid', []))
+        self.test = self.delete_oos(data.get('test', []))
+
+        self.train = self.get_shot_examples(self.train, shot)
+        self.valid = self.get_shot_examples(self.valid, shot_test)
+        self.test = self.get_shot_examples(self.test, shot_test)
+
+        save_path = Path(SUPPORT_DATASET_PATH).expanduser()
+        save_path.parent.mkdir(parents=True, exist_ok=True)
+        with save_path.open("w") as file:
+            json_dict = {
+                "columns": ["text", "category"],
+                "data": [[text, label] for text, label in self.train]
+            }
+            json.dump(json_dict, file, indent=4)
+
+        if return_nli_format:
+            self.train = self.convert2nli(self.train)
+            self.valid = self.convert2nli(self.valid)
+            self.test = self.convert2nli(self.test)
+
+        self.data = {
+            'train': self.train,
+            'valid': self.valid,
+            'test': self.test,
+            'all': self.train + self.test + self.valid
+        }
+
+    def _gather_info(self, data: List[Tuple[Any, Any]]) -> Tuple[Dict, Dict]:
+        unique_labels = list(set([label for text, label in data]))
+
+        label2examples = {}
+        for label in unique_labels:
+            label2examples[label] = []
+        for text, label in data:
+            label2examples[label].append(text)
+
+        label2negative = {}
+        for i, label in enumerate(unique_labels):
+            label2negative[label] = unique_labels.copy()
+            del label2negative[label][i]
+
+        return label2examples, label2negative
+
+    def convert2nli(self, data: List[Tuple[Any, Any]]) -> List[Tuple[Tuple[Any, Any], Any]]:
+        if len(data) == 0:
+            return data
+
+        label2examples, label2negative = self._gather_info(data)
+
+        nli_triplets = []
+        # negative examples
+        for text, label in data:
+            for negative_label in label2negative[label]:
+                for negative_example in label2examples[negative_label]:
+                    nli_triplets.append([[text, negative_example], NON_ENTAILMENT])
+
+        # positive examples
+        for text, label in data:
+            for positive_example in label2examples[label]:
+                if positive_example != text:
+                    nli_triplets.append([[text, positive_example], ENTAILMENT])
+
+        if self.shuffle:
+            self.random.shuffle(nli_triplets)
+
+        return nli_triplets
+
+    def delete_oos(self, data: List[Tuple[Any, Any]]) -> List[Tuple[Any, Any]]:
+        filtered_data = []
+        for text, label in data:
+            if label != 'oos':
+                filtered_data.append([text, label])
+        return filtered_data
+
+    def get_shot_examples(self, data: List[Tuple[Any, Any]], shot: int) -> List[Tuple[Any, Any]]:
+        if shot is None:
+            return data
+
+        # shuffle data to select shot-examples
+        self.random.shuffle(data)
+
+        data_dict = {}
+        for _, label in data:
+            data_dict[label] = []
+
+        for text, label in data:
+            if len(data_dict[label]) < shot:
+                data_dict[label].append(text)
+
+        if max(len(x) for x in data_dict.values()) < shot:
+            log.warning(f"Some labels have less than \"shot\"={shot} examples")
+
+        new_data = []
+        for label in data_dict.keys():
+            for text in data_dict[label]:
+                new_data.append((text, label))
+
+        if self.shuffle:
+            self.random.shuffle(new_data)
+
+        return new_data

deeppavlov/metrics/accuracy.py

@@ -14,7 +14,7 @@
 
 
 import itertools
-from typing import List, Iterable
+from typing import List
 
 import numpy as np
 
@@ -188,3 +188,16 @@ def kbqa_accuracy(y_true, y_predicted):
             total_correct += 1
 
     return total_correct / len(y_true) if len(y_true) else 0
+
+
+@register_metric('accuracy_oos')
+def accuracy_oos(y_true, y_pred, exclude_oos: bool = False) -> float:
+    if exclude_oos:
+        y_true = np.array(y_true)
+        y_pred = np.array(y_pred)
+
+        ind_mask = np.where(y_true == 'oos')
+
+        y_true = np.delete(y_true, ind_mask, 0)
+        y_pred = np.delete(y_pred, ind_mask, 0)
+    return accuracy(y_true, y_pred)

deeppavlov/metrics/fmeasure.py

@@ -19,6 +19,7 @@
 
 import numpy as np
 from sklearn.metrics import f1_score
+from sklearn.metrics import precision_recall_fscore_support
 
 from deeppavlov.core.common.metrics_registry import register_metric
 
@@ -417,3 +418,11 @@ def roc_auc__roc_auc__ner_f1(true_onehot1, pred_probas1, true_onehot2, pred_prob
     roc_auc2 = roc_auc_score(true_onehot2, pred_probas2)
     ner_f1_3 = ner_f1(ner_true3, ner_pred3) / 100
     return (roc_auc1 + roc_auc2 + ner_f1_3) / 3
+
+
+@register_metric('oos_scores')
+def oos_scores(y_true, y_pred):
+    y_true_binary = (np.array(y_true) == "oos")
+    y_pred_binary = (np.array(y_pred) == "oos")
+    scores = precision_recall_fscore_support(y_true_binary, y_pred_binary, average='binary')
+    return dict(zip(["precision", "recall", "fbeta_score"], scores[:3]))