(model): add hugging face transformer textattack/albert-base-v2-ag-news example

example/transformer/ag_news/README.md

@@ -0,0 +1,17 @@
+---
+title: Evaluate Hugging Face Transformer `textattack/albert-base-v2-ag-news` on AG News dataset
+---
+
+This example requires swcli version being above 0.4.0
+
+```bash
+$ python3 -m pip install starwhale>=0.4.0
+$ git clone https://github.com/star-whale/starwhale.git
+$ cd starwhale/example/text_cls_AG_NEWS
+$ swcli dataset build .
+$ cd ../transformer/ag_news/code
+$ python3 download_model.py
+$ cd ..
+$ python3 -m pip install -r requirements-sw-lock.txt
+$ swcli -vvv model eval  --dataset ag_news/version/latest .
+```

example/transformer/ag_news/code/download_model.py

@@ -0,0 +1,8 @@
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+
+tokenizer = AutoTokenizer.from_pretrained("textattack/albert-base-v2-ag-news")
+model = AutoModelForSequenceClassification.from_pretrained(
+    "textattack/albert-base-v2-ag-news"
+)
+tokenizer.save_pretrained("./ag_news/models")
+model.save_pretrained("./ag_news/models")

example/transformer/ag_news/code/evaluator.py

@@ -0,0 +1,39 @@
+from pathlib import Path
+
+from transformers import pipeline, AutoTokenizer, AutoModelForSequenceClassification
+
+from starwhale import PipelineHandler, multi_classification
+
+ROOTDIR = Path(__file__).parent.parent
+_LABEL_NAMES = ["LABEL_0", "LABEL_1", "LABEL_2", "LABEL_3"]
+
+
+class TextClassificationHandler(PipelineHandler):
+    def __init__(self) -> None:
+        super().__init__()
+        tokenizer = AutoTokenizer.from_pretrained(str(ROOTDIR / "models"))
+        model = AutoModelForSequenceClassification.from_pretrained(
+            str(ROOTDIR / "models")
+        )
+        self.mode = pipeline(
+            task="text-classification", model=model, tokenizer=tokenizer
+        )
+
+    def ppl(self, data: dict, **kw):
+        _r = self.mode(data["text"])
+        return _LABEL_NAMES.index(_r[0]["label"])
+
+    @multi_classification(
+        confusion_matrix_normalize="all",
+        show_hamming_loss=True,
+        show_cohen_kappa_score=True,
+        show_roc_auc=False,
+        all_labels=[i for i in range(0, 4)],
+    )
+    def cmp(self, ppl_result):
+        result, label = [], []
+        for _data in ppl_result:
+            label.append(_data["ds_data"]["label"])
+            result.append(_data["result"])
+
+        return label, result

example/transformer/ag_news/model.yaml

@@ -0,0 +1,15 @@
+version: 1.0
+name: ag_news_hg
+
+model:
+  - models/config.json
+  - models/pytorch_model.bin
+  - models/special_tokens_map.json
+  - models/spiece.model
+  - models/tokenizer.json
+  - models/tokenizer_config.json
+
+run:
+  handler: code.evaluator:TextClassificationHandler
+
+desc: TextClassification by huggingface transformer

example/transformer/ag_news/requirements-sw-lock.txt

@@ -0,0 +1,4 @@
+transformers
+torch
+sentencepiece
+protobuf==3.19.0

example/transformer/ag_news/runtime.yaml

@@ -0,0 +1,8 @@
+api_version: 1.1
+dependencies:
+  - requirements-sw-lock.txt
+environment:
+  arch: noarch
+  os: ubuntu:20.04
+mode: venv
+name: hg-textattack/albert-base-v2-ag-news