Skip to content

Commit

Permalink
chore (fix): calculate ner recall on cpu as it does not fit to GPU du…
Browse files Browse the repository at this point in the history 
…ring inference
  • Loading branch information
@vejvarm
vejvarm committed Dec 1, 2023
1 parent 10e5b8b commit 97970df
Show file tree
Hide file tree
Showing 4 changed files with 158 additions and 18 deletions.
105 changes: 105 additions & 0 deletions inference_for_ner_recall.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
import json
import pathlib
import random
import re
import numpy as np
from functools import partial

import pandas
from torch.utils.data import DataLoader
import torch
from tqdm import tqdm

from dataset import CSQADataset, collate_fn
from model import CARTON
from utils import Predictor, AverageMeter, MultiTaskAcc, MultiTaskAccTorchmetrics, MultiTaskRecTorchmetrics

from constants import DEVICE, LOGICAL_FORM, COREF, NER, INPUT, PREDICATE_POINTER, TYPE_POINTER, ROOT_PATH
from args import get_parser

parser = get_parser()
args = parser.parse_args()

random.seed(args.seed)
np.random.seed(args.seed)
if torch.cuda.is_available() and not args.no_cuda:
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)

DEVICE = f"{DEVICE}:{args.cuda_device}"
else:
DEVICE = "cpu"


def save_meter_to_file(meter_dict: dict[str: AverageMeter], path_to_file: pathlib.Path):
results = {nm: metric.avg.cpu().tolist() for nm, metric in meter_dict.items()}
results["average"] = np.mean([v for v in results.values()])

with path_to_file.open("w") as f:
json.dump(results, f, indent=4)


if __name__ == "__main__":
save_path = ROOT_PATH.joinpath(args.path_inference).joinpath(args.name)
print(f"results will be saved to `{save_path}`.")

# load data
dataset = CSQADataset(args, splits=('test', )) # assuming we already have the correct vocab cache from all splits!
data_dict, helper_dict = dataset.preprocess_data()
vocabs = dataset.build_vocabs(args.stream_data)

test_loader = torch.utils.data.DataLoader(data_dict['test'],
batch_size=args.batch_size,
shuffle=True,
collate_fn=partial(collate_fn, vocabs=vocabs, device=DEVICE))
total_batches = (len(test_loader.dataset) + args.batch_size - 1) // args.batch_size

pad = {k: v.stoi["[PAD]"] for k, v in vocabs.items() if k != "id"}
num_classes = {k: len(v) for k, v in vocabs.items() if k != "id"}

model = CARTON(vocabs, DEVICE).to(DEVICE)
print(f"=> loading checkpoint '{args.model_path}'")
checkpoint = torch.load(f'{ROOT_PATH}/{args.model_path}', encoding='latin1', map_location=DEVICE)
args.start_epoch = checkpoint['epoch']
model.load_state_dict(checkpoint['state_dict'])
print(f"=> loaded checkpoint '{args.model_path}' (epoch {checkpoint['epoch']})")

# RECALL Metric (Macro averaged) (except NER)
# !this is a HACK, we only calculate NER here on the CPU, because it does not fit on GPU
recall_modules = [NER]
recall_averaging_types = {mn: 'macro' for mn in recall_modules}
rec_calculator = MultiTaskRecTorchmetrics(num_classes, pads=pad, device='cpu',
averaging_types=recall_averaging_types, module_names=recall_modules)
recalls = {mn: AverageMeter() for mn in recall_modules}

with torch.no_grad():
with tqdm(total=total_batches, desc=f'Inference') as pbar:
for i, batch in enumerate(test_loader):
"""
Using model to do inference
"""

# compute output
output = model(batch.input, batch.logical_form[:, :-1])

target = {
LOGICAL_FORM: batch.logical_form[:, 1:].contiguous().view(-1),
NER: batch.ner.contiguous().view(-1),
COREF: batch.coref.contiguous().view(-1),
PREDICATE_POINTER: batch.predicate_pointer[:, 1:].contiguous().view(-1),
TYPE_POINTER: batch.type_pointer[:, 1:].contiguous().view(-1),
}

recs = rec_calculator({k: v.detach().cpu() for k, v in output.items()},
{k: v.detach().cpu() for k, v in target.items()})
for name, meter in recalls.items():
meter.update(recs[name])

pbar.set_postfix({'ner': f"{recalls[NER].avg:.4f}"})
pbar.update(1)

# save metric results
save_path.mkdir(exist_ok=True, parents=True)
path_to_rec = save_path.joinpath("rec-ner.json")
save_meter_to_file(recalls, path_to_rec)
10 changes: 7 additions & 3 deletions lab/get_vocab_examples.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,14 @@
if __name__ == "__main__":
vocab_folder = Path("../.cache/vocabs/")
pkl_list = list(vocab_folder.glob("*.pkl"))
vocab_example_map = {}
stats = {}
stats['sizes'] = {}
stats['examples'] = {}

for pth in pkl_list:
vocab = pickle.load(pth.open("rb"), encoding="utf8")
vocab_example_map[pth.stem] = [vocab.itos[i] for i in range(10)]
stats['sizes'][pth.stem] = len(vocab)
stats['examples'][pth.stem] = [vocab.itos[i] for i in range(10)]

json.dump(vocab_example_map, vocab_folder.joinpath("examples.json").open("w"))
json.dump(stats, vocab_folder.joinpath("stats.json").open("w"), indent=4)
print("done.")
30 changes: 15 additions & 15 deletions run_inference.sh
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,31 @@
#!/bin/bash

# CARTONNER trained on CSQA dataset for 10 epochs
python inference.py --name "00_csqa10_on_csqa" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/final/csqa --cuda-device 0
python inference.py --name "00_csqa10_on_csqa" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/final/csqa --cuda-device 0

python inference.py --name "00_csqa10_on_merged" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0
python inference.py --name "00_csqa10_on_merged" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0

python inference.py --name "00_csqa10_on_d2t" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0
python inference.py --name "00_csqa10_on_d2t" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0

# CARTONNER trained on MERGED dataset for 10 epochs
python inference.py --name "01_merged10_on_csqa" --batch-size 1 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/final/csqa --cuda-device 0
python inference.py --name "01_merged10_on_csqa" --batch-size 50 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/final/csqa --cuda-device 0

python inference.py --name "01_merged10_on_merged" --batch-size 1 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0
python inference.py --name "01_merged10_on_merged" --batch-size 50 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0

python inference.py --name "01_merged10_on_d2t" --batch-size 1 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0
python inference.py --name "01_merged10_on_d2t" --batch-size 50 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0


# CARTONNER trained on CSQA dataset for 15 epochs
python inference.py --name "02_csqa15_on_csqa" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/final/csqa --cuda-device 0
python inference.py --name "02_csqa15_on_csqa" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/final/csqa --cuda-device 0

python inference.py --name "02_csqa15_on_merged" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0
python inference.py --name "02_csqa15_on_merged" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0

python inference.py --name "02_csqa15_on_d2t" --batch-size 1 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0
python inference.py --name "02_csqa15_on_d2t" --batch-size 50 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0


## CARTONNER trained on MERGED dataset for 15 epochs
#python inference.py --name "03_merged15_on_csqa" --batch-size 1 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/final/csqa --cuda-device 0
#
#python inference.py --name "03_merged15_on_merged" --batch-size 1 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0
#
#python inference.py --name "03_merged15_on_d2t" --batch-size 1 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0
# CARTONNER trained on MERGED dataset for 15 epochs
python inference.py --name "03_merged15_on_csqa" --batch-size 50 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/final/csqa --cuda-device 0

python inference.py --name "03_merged15_on_merged" --batch-size 50 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/csqa-merged --cuda-device 0

python inference.py --name "03_merged15_on_d2t" --batch-size 50 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 0
31 changes: 31 additions & 0 deletions run_inference_for_ner.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
#!/bin/bash

# CARTONNER trained on CSQA dataset for 10 epochs
python inference_for_ner_recall.py --name "00_csqa10_on_csqa" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/final/csqa --cuda-device 1

python inference_for_ner_recall.py --name "00_csqa10_on_merged" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/csqa-merged --cuda-device 1

python inference_for_ner_recall.py --name "00_csqa10_on_d2t" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e10_v0.0154_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 1

# CARTONNER trained on MERGED dataset for 10 epochs
python inference_for_ner_recall.py --name "01_merged10_on_csqa" --batch-size 25 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/final/csqa --cuda-device 1

python inference_for_ner_recall.py --name "01_merged10_on_merged" --batch-size 25 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/csqa-merged --cuda-device 1

python inference_for_ner_recall.py --name "01_merged10_on_d2t" --batch-size 25 --model-path experiments/models/CARTONNER_merged15_e10_v0.0273_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 1


# CARTONNER trained on CSQA dataset for 15 epochs
python inference_for_ner_recall.py --name "02_csqa15_on_csqa" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/final/csqa --cuda-device 1

python inference_for_ner_recall.py --name "02_csqa15_on_merged" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/csqa-merged --cuda-device 1

python inference_for_ner_recall.py --name "02_csqa15_on_d2t" --batch-size 25 --model-path experiments/models/CARTONNER_csqa15_e15_v0.0148_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 1


# CARTONNER trained on MERGED dataset for 15 epochs
python inference_for_ner_recall.py --name "03_merged15_on_csqa" --batch-size 25 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/final/csqa --cuda-device 1

python inference_for_ner_recall.py --name "03_merged15_on_merged" --batch-size 25 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/csqa-merged --cuda-device 1

python inference_for_ner_recall.py --name "03_merged15_on_d2t" --batch-size 25 --model-path experiments/models/CARTONNER_merged_e15_v0.0146_multitask.pth.tar --data-path data/d2t-sampled --cuda-device 1

0 comments on commit 97970df

Please sign in to comment.