From 97970df403e76de8b8ad063440ebb3743dc5988f Mon Sep 17 00:00:00 2001 From: "vejvarm@freya" Date: Fri, 1 Dec 2023 21:08:25 +0900 Subject: [PATCH] chore (fix): calculate ner recall on cpu as it does not fit to GPU during inference --- inference_for_ner_recall.py | 105 ++++++++++++++++++++++++++++++++++++ lab/get_vocab_examples.py | 10 ++-- run_inference.sh | 30 +++++------ run_inference_for_ner.sh | 31 +++++++++++ 4 files changed, 158 insertions(+), 18 deletions(-) create mode 100644 inference_for_ner_recall.py create mode 100644 run_inference_for_ner.sh diff --git a/inference_for_ner_recall.py b/inference_for_ner_recall.py new file mode 100644 index 0000000..9c4a7ba --- /dev/null +++ b/inference_for_ner_recall.py @@ -0,0 +1,105 @@ +import json +import pathlib +import random +import re +import numpy as np +from functools import partial + +import pandas +from torch.utils.data import DataLoader +import torch +from tqdm import tqdm + +from dataset import CSQADataset, collate_fn +from model import CARTON +from utils import Predictor, AverageMeter, MultiTaskAcc, MultiTaskAccTorchmetrics, MultiTaskRecTorchmetrics + +from constants import DEVICE, LOGICAL_FORM, COREF, NER, INPUT, PREDICATE_POINTER, TYPE_POINTER, ROOT_PATH +from args import get_parser + +parser = get_parser() +args = parser.parse_args() + +random.seed(args.seed) +np.random.seed(args.seed) +if torch.cuda.is_available() and not args.no_cuda: + torch.manual_seed(args.seed) + torch.cuda.manual_seed(args.seed) + torch.cuda.manual_seed_all(args.seed) + + DEVICE = f"{DEVICE}:{args.cuda_device}" +else: + DEVICE = "cpu" + + +def save_meter_to_file(meter_dict: dict[str: AverageMeter], path_to_file: pathlib.Path): + results = {nm: metric.avg.cpu().tolist() for nm, metric in meter_dict.items()} + results["average"] = np.mean([v for v in results.values()]) + + with path_to_file.open("w") as f: + json.dump(results, f, indent=4) + + +if __name__ == "__main__": + save_path = ROOT_PATH.joinpath(args.path_inference).joinpath(args.name) + print(f"results will be saved to `{save_path}`.") + + # load data + dataset = CSQADataset(args, splits=('test', )) # assuming we already have the correct vocab cache from all splits! + data_dict, helper_dict = dataset.preprocess_data() + vocabs = dataset.build_vocabs(args.stream_data) + + test_loader = torch.utils.data.DataLoader(data_dict['test'], + batch_size=args.batch_size, + shuffle=True, + collate_fn=partial(collate_fn, vocabs=vocabs, device=DEVICE)) + total_batches = (len(test_loader.dataset) + args.batch_size - 1) // args.batch_size + + pad = {k: v.stoi["[PAD]"] for k, v in vocabs.items() if k != "id"} + num_classes = {k: len(v) for k, v in vocabs.items() if k != "id"} + + model = CARTON(vocabs, DEVICE).to(DEVICE) + print(f"=> loading checkpoint '{args.model_path}'") + checkpoint = torch.load(f'{ROOT_PATH}/{args.model_path}', encoding='latin1', map_location=DEVICE) + args.start_epoch = checkpoint['epoch'] + model.load_state_dict(checkpoint['state_dict']) + print(f"=> loaded checkpoint '{args.model_path}' (epoch {checkpoint['epoch']})") + + # RECALL Metric (Macro averaged) (except NER) + # !this is a HACK, we only calculate NER here on the CPU, because it does not fit on GPU + recall_modules = [NER] + recall_averaging_types = {mn: 'macro' for mn in recall_modules} + rec_calculator = MultiTaskRecTorchmetrics(num_classes, pads=pad, device='cpu', + averaging_types=recall_averaging_types, module_names=recall_modules) + recalls = {mn: AverageMeter() for mn in recall_modules} + + with torch.no_grad(): + with tqdm(total=total_batches, desc=f'Inference') as pbar: + for i, batch in enumerate(test_loader): + """ + Using model to do inference + """ + + # compute output + output = model(batch.input, batch.logical_form[:, :-1]) + + target = { + LOGICAL_FORM: batch.logical_form[:, 1:].contiguous().view(-1), + NER: batch.ner.contiguous().view(-1), + COREF: batch.coref.contiguous().view(-1), + PREDICATE_POINTER: batch.predicate_pointer[:, 1:].contiguous().view(-1), + TYPE_POINTER: batch.type_pointer[:, 1:].contiguous().view(-1), + } + + recs = rec_calculator({k: v.detach().cpu() for k, v in output.items()}, + {k: v.detach().cpu() for k, v in target.items()}) + for name, meter in recalls.items(): + meter.update(recs[name]) + + pbar.set_postfix({'ner': f"{recalls[NER].avg:.4f}"}) + pbar.update(1) + + # save metric results + save_path.mkdir(exist_ok=True, parents=True) + path_to_rec = save_path.joinpath("rec-ner.json") + save_meter_to_file(recalls, path_to_rec) diff --git a/lab/get_vocab_examples.py b/lab/get_vocab_examples.py index 968e878..ef6450e 100644 --- a/lab/get_vocab_examples.py +++ b/lab/get_vocab_examples.py @@ -8,10 +8,14 @@ if __name__ == "__main__": vocab_folder = Path("../.cache/vocabs/") pkl_list = list(vocab_folder.glob("*.pkl")) - vocab_example_map = {} + stats = {} + stats['sizes'] = {} + stats['examples'] = {} for pth in pkl_list: vocab = pickle.load(pth.open("rb"), encoding="utf8") - vocab_example_map[pth.stem] = [vocab.itos[i] for i in range(10)] + stats['sizes'][pth.stem] = len(vocab) + stats['examples'][pth.stem] = [vocab.itos[i] for i in range(10)] - json.dump(vocab_example_map, vocab_folder.joinpath("examples.json").open("w")) + json.dump(stats, vocab_folder.joinpath("stats.json").open("w"), indent=4) + print("done.") diff --git a/run_inference.sh b/run_inference.sh index c0f89e4..da4179d 100755 --- a/run_inference.sh +++ b/run_inference.sh @@ -1,31 +1,31 @@ #!/bin/bash # CARTONNER trained on CSQA dataset for 10 epochs -python inference.py --name "00_csqa10_on_csqa" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/final/csqa --cuda-device 0 +python inference.py --name "00_csqa10_on_csqa" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/final/csqa --cuda-device 0 -python inference.py --name "00_csqa10_on_merged" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0 +python inference.py --name "00_csqa10_on_merged" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0 -python inference.py --name "00_csqa10_on_d2t" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0 +python inference.py --name "00_csqa10_on_d2t" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0 # CARTONNER trained on MERGED dataset for 10 epochs -python inference.py --name "01_merged10_on_csqa" --batch-size 1 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/final/csqa --cuda-device 0 +python inference.py --name "01_merged10_on_csqa" --batch-size 50 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/final/csqa --cuda-device 0 -python inference.py --name "01_merged10_on_merged" --batch-size 1 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0 +python inference.py --name "01_merged10_on_merged" --batch-size 50 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0 -python inference.py --name "01_merged10_on_d2t" --batch-size 1 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0 +python inference.py --name "01_merged10_on_d2t" --batch-size 50 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0 # CARTONNER trained on CSQA dataset for 15 epochs -python inference.py --name "02_csqa15_on_csqa" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/final/csqa --cuda-device 0 +python inference.py --name "02_csqa15_on_csqa" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/final/csqa --cuda-device 0 -python inference.py --name "02_csqa15_on_merged" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0 +python inference.py --name "02_csqa15_on_merged" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0 -python inference.py --name "02_csqa15_on_d2t" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0 +python inference.py --name "02_csqa15_on_d2t" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0 -## CARTONNER trained on MERGED dataset for 15 epochs -#python inference.py --name "03_merged15_on_csqa" --batch-size 1 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/final/csqa --cuda-device 0 -# -#python inference.py --name "03_merged15_on_merged" --batch-size 1 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0 -# -#python inference.py --name "03_merged15_on_d2t" --batch-size 1 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0 \ No newline at end of file +# CARTONNER trained on MERGED dataset for 15 epochs +python inference.py --name "03_merged15_on_csqa" --batch-size 50 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/final/csqa --cuda-device 0 + +python inference.py --name "03_merged15_on_merged" --batch-size 50 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0 + +python inference.py --name "03_merged15_on_d2t" --batch-size 50 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0 \ No newline at end of file diff --git a/run_inference_for_ner.sh b/run_inference_for_ner.sh new file mode 100644 index 0000000..8fcc4f1 --- /dev/null +++ b/run_inference_for_ner.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# CARTONNER trained on CSQA dataset for 10 epochs +python inference_for_ner_recall.py --name "00_csqa10_on_csqa" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/final/csqa --cuda-device 1 + +python inference_for_ner_recall.py --name "00_csqa10_on_merged" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/csqa-merged --cuda-device 1 + +python inference_for_ner_recall.py --name "00_csqa10_on_d2t" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 1 + +# CARTONNER trained on MERGED dataset for 10 epochs +python inference_for_ner_recall.py --name "01_merged10_on_csqa" --batch-size 25 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/final/csqa --cuda-device 1 + +python inference_for_ner_recall.py --name "01_merged10_on_merged" --batch-size 25 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/csqa-merged --cuda-device 1 + +python inference_for_ner_recall.py --name "01_merged10_on_d2t" --batch-size 25 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 1 + + +# CARTONNER trained on CSQA dataset for 15 epochs +python inference_for_ner_recall.py --name "02_csqa15_on_csqa" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/final/csqa --cuda-device 1 + +python inference_for_ner_recall.py --name "02_csqa15_on_merged" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/csqa-merged --cuda-device 1 + +python inference_for_ner_recall.py --name "02_csqa15_on_d2t" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 1 + + +# CARTONNER trained on MERGED dataset for 15 epochs +python inference_for_ner_recall.py --name "03_merged15_on_csqa" --batch-size 25 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/final/csqa --cuda-device 1 + +python inference_for_ner_recall.py --name "03_merged15_on_merged" --batch-size 25 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/csqa-merged --cuda-device 1 + +python inference_for_ner_recall.py --name "03_merged15_on_d2t" --batch-size 25 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 1 \ No newline at end of file