Add PP-MiniLM

examples/model_compression/PP-MiniLM/data.py

@@ -0,0 +1,85 @@
+#   Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import numpy as np
+
+from paddle.metric import Metric, Accuracy
+from paddlenlp.transformers import ErnieForSequenceClassification, ErnieTokenizer
+from paddlenlp.transformers import BertForSequenceClassification, BertTokenizer
+
+MODEL_CLASSES = {
+    "ernie": (ErnieForSequenceClassification, ErnieTokenizer),
+    "bert": (BertForSequenceClassification, BertTokenizer)
+}
+
+METRIC_CLASSES = {
+    "afqmc": Accuracy,
+    "tnews": Accuracy,
+    "iflytek": Accuracy,
+    "ocnli": Accuracy,
+    "cmnli": Accuracy,
+    "cluewsc2020": Accuracy,
+    "csl": Accuracy,
+}
+
+
+def convert_example(example,
+                    tokenizer,
+                    label_list,
+                    max_seq_length=512,
+                    is_test=False):
+    """convert a glue example into necessary features"""
+    if not is_test:
+        # `label_list == None` is for regression task
+        label_dtype = "int64" if label_list else "float32"
+        # Get the label
+        label = example['label']
+        label = np.array([label], dtype=label_dtype)
+    # Convert raw text to feature
+    if 'sentence' in example:
+        example = tokenizer(example['sentence'], max_seq_len=max_seq_length)
+    elif 'sentence1' in example:
+        example = tokenizer(
+            example['sentence1'],
+            text_pair=example['sentence2'],
+            max_seq_len=max_seq_length)
+    elif 'keyword' in example:  # CSL
+        sentence1 = " ".join(example['keyword'])
+        example = tokenizer(
+            sentence1, text_pair=example['abst'], max_seq_len=max_seq_length)
+    elif 'target' in example:  # wsc
+        text, query, pronoun, query_idx, pronoun_idx = example['text'], example[
+            'target']['span1_text'], example['target']['span2_text'], example[
+                'target']['span1_index'], example['target']['span2_index']
+        text_list = list(text)
+        assert text[pronoun_idx:(pronoun_idx + len(pronoun)
+                                 )] == pronoun, "pronoun: {}".format(pronoun)
+        assert text[query_idx:(query_idx + len(query)
+                               )] == query, "query: {}".format(query)
+        if pronoun_idx > query_idx:
+            text_list.insert(query_idx, "_")
+            text_list.insert(query_idx + len(query) + 1, "_")
+            text_list.insert(pronoun_idx + 2, "[")
+            text_list.insert(pronoun_idx + len(pronoun) + 2 + 1, "]")
+        else:
+            text_list.insert(pronoun_idx, "[")
+            text_list.insert(pronoun_idx + len(pronoun) + 1, "]")
+            text_list.insert(query_idx + 2, "_")
+            text_list.insert(query_idx + len(query) + 2 + 1, "_")
+        text = "".join(text_list)
+        example = tokenizer(text, max_seq_len=max_seq_length)
+
+    if not is_test:
+        return example['input_ids'], example['token_type_ids'], label
+    else:
+        return example['input_ids'], example['token_type_ids']

examples/model_compression/PP-MiniLM/finetuning/export_model.py

@@ -0,0 +1,78 @@
+# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import os
+
+import paddle
+
+from run_clue import MODEL_CLASSES
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+
+    # Required parameters
+    parser.add_argument(
+        "--model_type",
+        default=None,
+        type=str,
+        required=True,
+        help="Model type selected in the list: " +
+        ", ".join(MODEL_CLASSES.keys()), )
+    parser.add_argument(
+        "--model_path",
+        default=None,
+        type=str,
+        required=True,
+        help="Path of the trained model to be exported.", )
+    parser.add_argument(
+        "--output_path",
+        default=None,
+        type=str,
+        required=True,
+        help="The output file prefix used to save the exported inference model.",
+    )
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+
+    args.model_type = args.model_type.lower()
+    model_class, tokenizer_class = MODEL_CLASSES[args.model_type]
+
+    # build model and load trained parameters
+    model = model_class.from_pretrained(args.model_path)
+    # switch to eval model
+    model.eval()
+    # convert to static graph with specific input description
+    model = paddle.jit.to_static(
+        model,
+        input_spec=[
+            paddle.static.InputSpec(
+                shape=[None, None], dtype="int64"),  # input_ids
+            paddle.static.InputSpec(
+                shape=[None, None], dtype="int64")  # segment_ids
+        ])
+    # save converted static graph model
+    paddle.jit.save(model, args.output_path)
+    # also save tokenizer for inference usage
+    tokenizer = tokenizer_class.from_pretrained(args.model_path)
+    tokenizer.save_pretrained(os.path.dirname(args.output_path))
+
+
+if __name__ == "__main__":
+    main()

examples/model_compression/PP-MiniLM/finetuning/run_all_search.sh

@@ -0,0 +1,35 @@
+# $1 means GENERAL_DIR
+
+# The penultimate parameter is the card id, this script can be changed if necessary
+bash run_one_search.sh $1 afqmc 0 &
+bash run_one_search.sh $1 tnews 1 &
+bash run_one_search.sh $1 ifly 2 &
+bash run_one_search.sh $1 ocnli 3 &
+bash run_one_search.sh $1 csl 4 &
+bash run_one_search.sh $1 wsc 5 &
+
+# Because the CMNLI data set is significantly larger than other data sets,
+# it needs to be placed on different cards.
+lr=1e-4
+bs=16
+sh run_clue.sh CMNLI $lr $bs 3 128 0 $1  > $1/cmnli/${lr}_${bs}_3_128.log &
+bs=32
+sh run_clue.sh CMNLI $lr $bs 3 128 1 $1  > $1/cmnli/${lr}_${bs}_3_128.log &
+bs=64
+sh run_clue.sh CMNLI $lr $bs 3 128 2 $1  > $1/cmnli/${lr}_${bs}_3_128.log &
+
+lr=5e-5
+bs=16
+sh run_clue.sh CMNLI $lr $bs 3 128 3 $1  > $1/cmnli/${lr}_${bs}_3_128.log &
+bs=32
+sh run_clue.sh CMNLI $lr $bs 3 128 4 $1  > $1/cmnli/${lr}_${bs}_3_128.log &
+bs=64
+sh run_clue.sh CMNLI $lr $bs 3 128 5 $1  > $1/cmnli/${lr}_${bs}_3_128.log &
+
+lr=3e-5
+bs=16
+sh run_clue.sh CMNLI $lr $bs 3 128 6 $1  > $1/cmnli/${lr}_${bs}_3_128.log &
+bs=32
+sh run_clue.sh CMNLI $lr $bs 3 128 5 $1  > $1/cmnli/${lr}_${bs}_3_128.log &
+bs=64
+sh run_clue.sh CMNLI $lr $bs 3 128 7 $1  > $1/cmnli/${lr}_${bs}_3_128.log &