Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

tweet cleanup (with potentational peformance boost) #22

Open
wants to merge 16 commits into
base: main
Choose a base branch
from
Open
2 changes: 1 addition & 1 deletion .flake8
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
[flake8]
exclude = venv
ignore = W503 #line break occurred before binary operation
ignore = W503,W605 #line break occurred before binary operation
Copy link
Owner Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

otherwise we get complaints about our regexes in replacement_patterns

max-line-length = 100
7 changes: 2 additions & 5 deletions config/model/model_config.yaml
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
lr: 6e-6
eps: 1e-8
# model: 'bert'
# pretrained-model: 'bert-large-uncased'
model: 'distilbert'
model: 'bert'
#model: 'distilbert'
pretrained-model: 'distilbert-base-uncased'
num_labels: 2
11 changes: 5 additions & 6 deletions config/train/train_config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
optimizer: Adam
lr: 0.001
batch_size: 8
scheduler:
name: ExponentialLR
gamma: 0.1
optimizer: AdamW
lr: 6e-6
eps: 1e-8,
batch_size: 16
epochs: 5
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ torch==1.10.1
transformers==4.15.0
google-cloud-secret-manager==2.5.0
wandb==0.12.9
nltk==3.6.7
9 changes: 9 additions & 0 deletions src/features/build_features.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@
from pathlib import Path

import hydra
import nltk
import numpy as np
import pandas as pd # type: ignore
import torch
from dotenv import find_dotenv, load_dotenv
from omegaconf import DictConfig
from transformers import AutoTokenizer
from tweet_cleaner import clean_tweet_list

# See kaggle notebook:
# https://www.kaggle.com/gunesevitan/nlp-with-disaster-tweets-eda-cleaning-and-bert
Expand Down Expand Up @@ -46,6 +48,13 @@ def main(cfg: DictConfig) -> None:
list(data.target[split_eval:]),
)

# %% Clean
nltk.download("wordnet")
nltk.download("omw-1.4")
tweet_train = clean_tweet_list(tweet_train)
tweet_test = clean_tweet_list(tweet_test)
tweet_eval = clean_tweet_list(tweet_eval)

# %% Encode
tokenizer = AutoTokenizer.from_pretrained(cfg.model["pretrained-model"])

Expand Down
305 changes: 305 additions & 0 deletions src/features/tweet_cleaner.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,305 @@
import re

import nltk

replacement_patterns = [
(r"won\'t", "will not"),
(r"can\'t", "cannot"),
(r"i\'m", "i am"),
(r"ain\'t", "is not"),
(r"(\w+)\'ll", "\g<1> will"),
(r"(\w+)n\'t", "\g<1> not"),
(r"(\w+)\'ve", "\g<1> have"),
(r"(\w+)\'s", "\g<1> is"),
(r"(\w+)\'re", "\g<1> are"),
(r"(\w+)\'d", "\g<1> would"),
]
abbreviations = {
"$": " dollar ",
"€": " euro ",
"4ao": "for adults only",
"a.m": "before midday",
"a3": "anytime anywhere anyplace",
"aamof": "as a matter of fact",
"acct": "account",
"adih": "another day in hell",
"afaic": "as far as i am concerned",
"afaict": "as far as i can tell",
"afaik": "as far as i know",
"afair": "as far as i remember",
"afk": "away from keyboard",
"app": "application",
"approx": "approximately",
"apps": "applications",
"asap": "as soon as possible",
"asl": "age, sex, location",
"atk": "at the keyboard",
"ave.": "avenue",
"aymm": "are you my mother",
"ayor": "at your own risk",
"b&b": "bed and breakfast",
"b+b": "bed and breakfast",
"b.c": "before christ",
"b2b": "business to business",
"b2c": "business to customer",
"b4": "before",
"b4n": "bye for now",
"b@u": "back at you",
"bae": "before anyone else",
"bak": "back at keyboard",
"bbbg": "bye bye be good",
"bbc": "british broadcasting corporation",
"bbias": "be back in a second",
"bbl": "be back later",
"bbs": "be back soon",
"be4": "before",
"bfn": "bye for now",
"blvd": "boulevard",
"bout": "about",
"brb": "be right back",
"bros": "brothers",
"brt": "be right there",
"bsaaw": "big smile and a wink",
"btw": "by the way",
"bwl": "bursting with laughter",
"c/o": "care of",
"cet": "central european time",
"cf": "compare",
"cia": "central intelligence agency",
"csl": "can not stop laughing",
"cu": "see you",
"cul8r": "see you later",
"cv": "curriculum vitae",
"cwot": "complete waste of time",
"cya": "see you",
"cyt": "see you tomorrow",
"dae": "does anyone else",
"dbmib": "do not bother me i am busy",
"diy": "do it yourself",
"dm": "direct message",
"dwh": "during work hours",
"e123": "easy as one two three",
"eet": "eastern european time",
"eg": "example",
"embm": "early morning business meeting",
"encl": "enclosed",
"encl.": "enclosed",
"etc": "and so on",
"faq": "frequently asked questions",
"fawc": "for anyone who cares",
"fb": "facebook",
"fc": "fingers crossed",
"fig": "figure",
"fimh": "forever in my heart",
"ft.": "feet",
"ft": "featuring",
"ftl": "for the loss",
"ftw": "for the win",
"fwiw": "for what it is worth",
"fyi": "for your information",
"g9": "genius",
"gahoy": "get a hold of yourself",
"gal": "get a life",
"gcse": "general certificate of secondary education",
"gfn": "gone for now",
"gg": "good game",
"gl": "good luck",
"glhf": "good luck have fun",
"gmt": "greenwich mean time",
"gmta": "great minds think alike",
"gn": "good night",
"g.o.a.t": "greatest of all time",
"goat": "greatest of all time",
"goi": "get over it",
"gps": "global positioning system",
"gr8": "great",
"gratz": "congratulations",
"gyal": "girl",
"h&c": "hot and cold",
"hp": "horsepower",
"hr": "hour",
"hrh": "his royal highness",
"ht": "height",
"ibrb": "i will be right back",
"ic": "i see",
"icq": "i seek you",
"icymi": "in case you missed it",
"idc": "i do not care",
"idgadf": "i do not give a damn fuck",
"idgaf": "i do not give a fuck",
"idk": "i do not know",
"ie": "that is",
"i.e": "that is",
"ifyp": "i feel your pain",
"IG": "instagram",
"iirc": "if i remember correctly",
"ilu": "i love you",
"ily": "i love you",
"imho": "in my humble opinion",
"imo": "in my opinion",
"imu": "i miss you",
"iow": "in other words",
"irl": "in real life",
"j4f": "just for fun",
"jic": "just in case",
"jk": "just kidding",
"jsyk": "just so you know",
"l8r": "later",
"lb": "pound",
"lbs": "pounds",
"ldr": "long distance relationship",
"lmao": "laugh my ass off",
"lmfao": "laugh my fucking ass off",
"lol": "laughing out loud",
"ltd": "limited",
"ltns": "long time no see",
"m8": "mate",
"mf": "motherfucker",
"mfs": "motherfuckers",
"mfw": "my face when",
"mofo": "motherfucker",
"mph": "miles per hour",
"mr": "mister",
"mrw": "my reaction when",
"ms": "miss",
"mte": "my thoughts exactly",
"nagi": "not a good idea",
"nbc": "national broadcasting company",
"nbd": "not big deal",
"nfs": "not for sale",
"ngl": "not going to lie",
"nhs": "national health service",
"nrn": "no reply necessary",
"nsfl": "not safe for life",
"nsfw": "not safe for work",
"nth": "nice to have",
"nvr": "never",
"nyc": "new york city",
"oc": "original content",
"og": "original",
"ohp": "overhead projector",
"oic": "oh i see",
"omdb": "over my dead body",
"omg": "oh my god",
"omw": "on my way",
"p.a": "per annum",
"p.m": "after midday",
"pm": "prime minister",
"poc": "people of color",
"pov": "point of view",
"pp": "pages",
"ppl": "people",
"prw": "parents are watching",
"ps": "postscript",
"pt": "point",
"ptb": "please text back",
"pto": "please turn over",
"qpsa": "what happens", # "que pasa",
"ratchet": "rude",
"rbtl": "read between the lines",
"rlrt": "real life retweet",
"rofl": "rolling on the floor laughing",
"roflol": "rolling on the floor laughing out loud",
"rotflmao": "rolling on the floor laughing my ass off",
"rt": "retweet",
"ruok": "are you ok",
"sfw": "safe for work",
"sk8": "skate",
"smh": "shake my head",
"sq": "square",
"srsly": "seriously",
"ssdd": "same stuff different day",
"tbh": "to be honest",
"tbs": "tablespooful",
"tbsp": "tablespooful",
"tfw": "that feeling when",
"thks": "thank you",
"tho": "though",
"thx": "thank you",
"tia": "thanks in advance",
"til": "today i learned",
"tl;dr": "too long i did not read",
"tldr": "too long i did not read",
"tmb": "tweet me back",
"tntl": "trying not to laugh",
"ttyl": "talk to you later",
"u": "you",
"u2": "you too",
"u4e": "yours for ever",
"utc": "coordinated universal time",
"w/": "with",
"w/o": "without",
"w8": "wait",
"wassup": "what is up",
"wb": "welcome back",
"wtf": "what the fuck",
"wtg": "way to go",
"wtpa": "where the party at",
"wuf": "where are you from",
"wuzup": "what is up",
"wywh": "wish you were here",
"yd": "yard",
"ygtr": "you got that right",
"ynk": "you never know",
"zzz": "sleeping bored and tired",
}


class RegexpReplacer(object):
# Replaces regular expression in a text.
def __init__(self, patterns=replacement_patterns):
self.patterns = [(re.compile(regex), repl) for (regex, repl) in patterns]

def replace(self, text):
s = text

for (pattern, repl) in self.patterns:
s = re.sub(pattern, repl, s)
return s


def convert_abbrev(word):
return abbreviations[word.lower()] if word.lower() in abbreviations.keys() else word


def clean_tweet(text: str):
# remove urls
# text = df.apply(lambda x: re.sub(r'http\S+', '', x))
text = re.sub(r"http\S+", "", text)

# replace contractions
replacer = RegexpReplacer()
text = replacer.replace(text)

# split words on - and \
text = re.sub(r"\b", " ", text)
text = re.sub(r"-", " ", text)
# replace negations with antonyms

# nltk.download('punkt')
tokenizer = nltk.RegexpTokenizer(r"\w+")
tokens = tokenizer.tokenize(text)

# Replace abbreviations
tokens = [convert_abbrev(word) for word in tokens]

# todo: spelling correction
# replacer = SpellingReplacer()
# tokens = [replacer.replace(t) for t in tokens]

# lemmatize/stemming
wnl = nltk.WordNetLemmatizer()
tokens = [wnl.lemmatize(t) for t in tokens]

# todo: stemming conflicts with our tokenizer (Bert)
# porter = nltk.PorterStemmer()
# tokens = [porter.stem(t) for t in tokens]
# filter insignificant words (using fastai)
# swap word phrases

text = " ".join(tokens)
return text


def clean_tweet_list(tweet_list: list[str]):
return list(map(clean_tweet, tweet_list))
12 changes: 8 additions & 4 deletions src/models/model.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,13 @@ def __init__(self, config: DictConfig):
output_attentions=False,
output_hidden_states=False,
)
else: # default model is distilbert
elif (
self.config.model["model"] == "distilbert-base-uncased"
denisramiros marked this conversation as resolved.
Show resolved Hide resolved
): # default model is distilbert
print("Using DistilBert")
self.model = DistilBertForSequenceClassification.from_pretrained(
self.config.model["pretrained-model"],
num_labels=self.config.model["num_labels"],
output_attentions=False,
output_hidden_states=False,
)

def forward(self, inputs):
Expand Down Expand Up @@ -69,7 +70,10 @@ def configure_optimizers( # noqa: C901
) -> tuple[list[torch.optim.Optimizer], list[object]]:
if self.config.train["optimizer"] == "AdamW":
optimizer = torch.optim.AdamW(
self.parameters(), lr=self.config.train["lr"]
self.parameters(),
lr=self.config.train["lr"],
eps=self.config.train["eps"],
betas=(0.9, 0.999),
) # type: torch.optim.Optimizer
elif self.config.train["optimizer"] == "Adam":
optimizer = torch.optim.Adam(self.parameters(), lr=self.config.train["lr"])
Expand Down
Loading