data_cleanup.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ALlROGR5QPBP"
      },
      "source": [
        "# Project about 'en'\n",
        "Script for converting PDF to plain text file format, and cleaning up data to remove unnecessary annotations"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "y10dJ-YjQiOI"
      },
      "source": [
        "## Converting PDF to TXT format"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "3Snrq90WQKNf"
      },
      "outputs": [],
      "source": [
        "# PDF converter\n",
        "!pip install PyPDF2"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "e3RoeXXKQyhJ",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "74a8b193-c377-4ebc-c025-d1e4e33f7e70"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "# run this code if connecting to a Google drive\n",
        "from google.colab import drive\n",
        "\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import PyPDF2\n",
        "import re\n",
        "import os\n",
        "import pandas as pd"
      ],
      "metadata": {
        "id": "N9tztFeUdmbD"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "PaE8QBZxQ_Gx"
      },
      "outputs": [],
      "source": [
        "def pdf_to_txt(pdf_path, output_txt):\n",
        "    # Open the PDF file in read-binary mode\n",
        "    with open(pdf_path, 'rb') as pdf_file:\n",
        "        # Create a PdfReader object instead of PdfFileReader\n",
        "        pdf_reader = PyPDF2.PdfReader(pdf_file)\n",
        "\n",
        "        # Initialize an empty string to store the text\n",
        "        text = ''\n",
        "\n",
        "        for page_num in range(len(pdf_reader.pages)):\n",
        "            page = pdf_reader.pages[page_num]\n",
        "            text += page.extract_text()\n",
        "\n",
        "    # Write the extracted text to a text file\n",
        "    with open(output_txt, 'w', encoding='utf-8') as txt_file:\n",
        "        txt_file.write(text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "52knseGmS8p6"
      },
      "outputs": [],
      "source": [
        "# take pdf files from the corpus and convert them to txt files\n",
        "\n",
        "corpus = '/content/drive/My Drive/en_project/corpus'\n",
        "\n",
        "txt_dir = '/content/drive/My Drive/en_project/txt_files'\n",
        "\n",
        "input = ''\n",
        "output = ''\n",
        "\n",
        "for f in os.scandir(corpus):\n",
        "  if f.is_file():\n",
        "    input = f.path\n",
        "    filename = os.path.basename(f)\n",
        "    filename = re.sub(r'.pdf', '.txt', filename)\n",
        "    output = os.path.join(txt_dir, filename)\n",
        "\n",
        "    pdf_to_txt(input, output)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HelfbexjTbVy"
      },
      "source": [
        "## Cleaning up plain text files"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {
        "id": "1aLnLrYQubLo"
      },
      "outputs": [],
      "source": [
        "# annotation conventions: https://applis.flsh.usherbrooke.ca/cfpq/index.php/site/afficher/verbal\n",
        "\n",
        "def clean_data(old_file, new_file):\n",
        "\n",
        "  with open(old_file, 'r', encoding='utf-8') as txt_file:\n",
        "\n",
        "    text = txt_file.read()\n",
        "\n",
        "    # remove carriage returns\n",
        "    text = re.sub(r'\\n+', ' ', text)\n",
        "\n",
        "    # remove extraneous content\n",
        "    text = re.sub(r'\\(.*?\\)', '', text) # removes all content in parentheses\n",
        "    text = re.sub(r'\\[\\s*\\d', '', text) # removes all overlap indicators\n",
        "    text = re.sub(r';', '', text) # removes semicolons\n",
        "    text = re.sub(r'(?<=[a-zA-ZÀ-ÿ:]):', '', text) # removes random colons\n",
        "    text = re.sub(r'<([a-zA-Z]+)<', '', text) # removes volume and speed markers\n",
        "    text = re.sub(r'<.*?>', '', text) # removes everything in angle brackets\n",
        "    text = re.sub(r'[\\\\/{}<>\\u00B0\\u2022\\u2191\\u2193\\(\\)]', '', text) # removes slashes, all bracket types, bullets, arrows\n",
        "    text = re.sub(r'SOUS\\s*\\-\\s*CORPUS\\s*\\d* : segment \\d*\\.*\\d*', '', text) # removes the specific SOUS-CORPUS markings every page\n",
        "    text = re.sub(r'\\d*', '', text) # remove any numbers that were missed\n",
        "\n",
        "    # removes extra spaces after deleting the extra markers\n",
        "    text = re.sub(r'\\s{2,}', ' ', text)\n",
        "    text = re.sub(r'([A-ZÀ-Ü]* :)', r'\\n\\1', text)\n",
        "\n",
        "    text = text.strip() # clean up trailing whitespaces\n",
        "\n",
        "    with open(new_file, 'w', encoding='utf-8') as txt_file:\n",
        "      txt_file.write(text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "id": "eS7jXj8nudXi"
      },
      "outputs": [],
      "source": [
        "# take raw txt files from the corpus and clean them up\n",
        "dataset = '/content/drive/My Drive/en_project/txt_files'\n",
        "\n",
        "txt_dir = '/content/drive/My Drive/en_project/cleaned_txt_files'\n",
        "\n",
        "input = ''\n",
        "output = ''\n",
        "\n",
        "for f in os.scandir(dataset):\n",
        "  if f.is_file():\n",
        "    input = f.path\n",
        "    filename = os.path.basename(f)\n",
        "    filename = re.sub(r'.txt', '_clean.txt', filename)\n",
        "    output = os.path.join(txt_dir, filename)\n",
        "\n",
        "    clean_data(input, output)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Finding 'en'\n",
        "Find instances of 'en' in text file and output them to a spreadsheet with the following headers:\n",
        "* File name\n",
        "* Text\n",
        "* Information status\n",
        "* Il y a en?\n",
        "* Precise quantity?\n",
        "* Anaphor?\n",
        "* Answer to a question?\n",
        "* Polarity?\n",
        "* Locative?\n",
        "* Comments"
      ],
      "metadata": {
        "id": "tgaD4bsAUTI9"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# helper function for finding instances of en\n",
        "def find_en(file_path, file_name, df):\n",
        "  with open(file_path, 'r', encoding='utf-8') as clean_text:\n",
        "\n",
        "    lines = clean_text.readlines()\n",
        "\n",
        "    for line in lines:\n",
        "      if re.search(r'\\ben\\b', line):\n",
        "        row = {\n",
        "            'file_name': re.sub(r'_clean.txt', '', file_name),\n",
        "            'text': line,\n",
        "            'info': '',\n",
        "            'exists': '',\n",
        "            'quantity': '',\n",
        "            'anaphor': '',\n",
        "            'answer': '',\n",
        "            'polarity': '',\n",
        "            'locative': '',\n",
        "            'comments': ''\n",
        "        }\n",
        "\n",
        "        df.loc[len(df)] = row"
      ],
      "metadata": {
        "id": "-VnrUjjYgqrN"
      },
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# for each file\n",
        "# scan through the file for instances of 'en'\n",
        "# if found, save the entire instance (anything between two speaker patterns) as text\n",
        "# and extract file name as well\n",
        "# remaining columns remain empty strings\n",
        "\n",
        "annotations_df = pd.DataFrame(columns = ['file_name',\n",
        "                                         'text',\n",
        "                                         'info',\n",
        "                                         'exists',\n",
        "                                         'quantity',\n",
        "                                         'anaphor',\n",
        "                                         'answer',\n",
        "                                         'polarity',\n",
        "                                         'locative',\n",
        "                                         'comments'])\n",
        "\n",
        "cleaned = '/content/drive/My Drive/en_project/cleaned_txt_files'\n",
        "file_path = ''\n",
        "\n",
        "for f in os.scandir(cleaned):\n",
        "  file_path = f.path\n",
        "  file_name = os.path.basename(f)\n",
        "  find_en(file_path, file_name, annotations_df)"
      ],
      "metadata": {
        "id": "ft5C_Ot5VB-C"
      },
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "annotations_df.head()"
      ],
      "metadata": {
        "id": "sz8AG-9Fbpb2"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# rename column headers to prepare for writing to excel sheet\n",
        "annotations_df.rename(columns = {'file_name': 'File name',\n",
        "                                 'text': 'Clean text',\n",
        "                                 'info': 'Information status',\n",
        "                                 'exists': 'Il y a en?',\n",
        "                                 'quantity': 'Precise quantity?',\n",
        "                                 'anaphor': 'Anaphor?',\n",
        "                                 'answer': 'Answer to a question?',\n",
        "                                 'polarity': 'Polarity',\n",
        "                                 'locative': 'Locative?',\n",
        "                                 'comments': 'Comments'\n",
        "                                 }, inplace=True)"
      ],
      "metadata": {
        "id": "RcKabNzVisDJ"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# write to excel sheet\n",
        "annotations_file = '/content/drive/My Drive/en_project/annotations.xlsx'\n",
        "\n",
        "annotations_df.to_excel(annotations_file, sheet_name='Sheet1', index=False )"
      ],
      "metadata": {
        "id": "tsuBsRfGi1w3"
      },
      "execution_count": 39,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}