locationUpdation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5d7b91b",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "730583b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"./../NewDataSet0.csv\")\n",
    "print(dataset[\"location\"].size)\n",
    "dataset = dataset.dropna(subset=[\"location\"]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d17b5701",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from stop_words import get_stop_words\n",
    "from nltk.tokenize import word_tokenize\n",
    "\n",
    "# Load stopwords\n",
    "stop_words = get_stop_words('english')\n",
    "\n",
    "# Function to tokenize and remove stopwords\n",
    "def tokenize_and_remove_stopwords(text):\n",
    "    tokens = word_tokenize(text)  # Tokenize the text\n",
    "    filtered_tokens = [token for token in tokens if token.lower() not in stop_words]  # Remove stopwords\n",
    "    return filtered_tokens\n",
    "\n",
    "# Tokenize and remove stopwords for 'location' column\n",
    "dataset['location'] = dataset['location'].apply(tokenize_and_remove_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f130cef",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Function to check if a city or country is mentioned in the text and return the corresponding country\n",
    "def find_country_in_text(text, df,df1):\n",
    "    for word in text:\n",
    "        wd = word.lower()\n",
    "        print(wd)\n",
    "        v = df1.iloc[1:,1].searchsorted(wd)\n",
    "        if(v == 44690):\n",
    "            continue\n",
    "        if wd == df1.iloc[v+1,1]:\n",
    "            return wd\n",
    "    for word in text:\n",
    "        wd = word.lower()\n",
    "        v = df.iloc[1:,0].searchsorted(wd)\n",
    "        if(v == 44690):\n",
    "            continue\n",
    "        if wd == df.iloc[v+1,0]:\n",
    "            return df.iloc[v+1,1]\n",
    "    return \"none\"\n",
    "\n",
    "# Example usage\n",
    "df = pd.read_csv(\"./../worldcities.csv\",usecols = [1,4])\n",
    "df.sort_values(\"city_ascii\", inplace=True)\n",
    "df = df.reset_index(drop=True)\n",
    "df[\"city_ascii\"] = df[\"city_ascii\"].apply(lambda x: x.lower())\n",
    "df[\"country\"] = df[\"country\"].apply(lambda x: x.lower())\n",
    "\n",
    "df1 = pd.read_csv(\"./../worldcities.csv\",usecols = [1,4])\n",
    "df1.sort_values(\"country\", inplace=True)\n",
    "df1 = df1.reset_index(drop=True)\n",
    "df1[\"city_ascii\"] = df1[\"city_ascii\"].apply(lambda x: x.lower())\n",
    "df1[\"country\"] = df1[\"country\"].apply(lambda x: x.lower())\n",
    "\n",
    "dataset[\"location\"] = dataset[\"location\"].apply(lambda x: find_country_in_text(x, df,df1))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d937822",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = dataset[dataset['location'] != 'none']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b598ecda",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dataset[\"location\"].size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3ee25f69",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dataset[\"location\"].size)\n",
    "file_path = './loc_data.csv'  # Replace with the actual file path\n",
    "if os.path.exists(file_path):\n",
    "    old_data = pd.read_csv(\"loc_data.csv\")\n",
    "    old_data = pd.concat([old_data, dataset])\n",
    "    old_data.to_csv('loc_data.csv', sep=',', index=False)\n",
    "else:\n",
    "    dataset.to_csv('loc_data.csv', sep=',', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccda78be",
   "metadata": {},
   "outputs": [],
   "source": [
    "ans = 0;\n",
    "for i in app:\n",
    "    if(i != \"none\"):\n",
    "        ans = ans + 1;\n",
    "print(ans)\n",
    "print(app.unique().size)\n",
    "ct = app.unique()\n",
    "for i in ct:\n",
    "    print(i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dd19c683",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = pd.read_csv(\"./ConcData.csv\")\n",
    "print(dataset[\"location\"].unique())\n",
    "op = dataset[\"location\"].unique()\n",
    "m = {'united states' : 0, 'canada' : 0, 'united kingdom' : 0, 'japan' : 0, 'australia' : 0, 'india' : 0, 'nigeria' : 0, 'puerto rico' : 0, 'philippines' : 0, 'mali' : 0}\n",
    "for i in dataset[\"location\"]:\n",
    "    m[i] = m[i] + 1\n",
    "print(m)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}