CELT/CELT_Crawler.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vH86q4xzpbt1"
   },
   "source": [
    "# A crawler for Irish text from UCC CELT\n",
    "\n",
    "Requires list of pages to download as input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "IGkx0PeIpbuE"
   },
   "outputs": [],
   "source": [
    "import re\n",
    "import time\n",
    "import requests\n",
    "import pandas as pd\n",
    "import pickle as pkl\n",
    "from tqdm import tqdm\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib3.exceptions import MaxRetryError, NewConnectionError\n",
    "from requests.exceptions import ConnectionError"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IhfhCledrKkg",
    "outputId": "ba5edbe0-88a9-4733-a95e-37270d18de8b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "CiCO3FUPrYzS",
    "outputId": "72c84b89-cd0a-471f-b721-53d753a71aff"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/content/drive/My Drive/Colab Notebooks/medieval-embeddings\n"
     ]
    }
   ],
   "source": [
    "%cd drive/My Drive/Colab Notebooks/medieval-embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "id": "lImOf1cYpbuI"
   },
   "outputs": [],
   "source": [
    "punc = \"[—‘’:;.,!?–&°£§$\\\"^«»<>/\\|%=\\[\\]()'*+]+\"\n",
    "\n",
    "entry_pattern = \"\"\"\n",
    "{\"Title\":\"%s\", \"Editor\":\"%s\", \"Size (words)\":\"%s\", \"Real size (tokens)\":\"%s\", \"Unique forms\":\"%s\", \"Dates\":\"%s\", \"Irish\":\"%s\", \"Other languages\":\"%s\", \"Form\":\"%s\", \"Genre\":\"%s\", \"Subgenre\":\"%s\", \"Group\":\"—\", \"Author\":\"%s\", \"Link\":\"%s\", \"Comment\":\"—\", \"Raw Text\":\"–\", \"Text\":\"–\", \"Preprocessed Text\":\"–\"}\n",
    "\"\"\"\n",
    "\n",
    "columns = [\"Title\", \"Editor\", \"Size (words)\", \"Real size (tokens)\", \"Unique forms\", \"Dates\", \"Irish\", \n",
    "           \"Other languages\", \"Form\", \"Genre\", \"Subgenre\", \"Group\", \"Author\", \"Link\", \"Comment\", \"Raw Text\",\n",
    "           \"Text\", \"Preprocessed Text\"]\n",
    "\n",
    "mutations = {\n",
    "                'bh': 'b', 'mb': 'b', 'ch': 'c', 'gc': 'c', 'gh': 'g',\n",
    "                'ng': 'g', 'dh': 'd', 'nd': 'd', 'fh': 'f', 'ḟ': 'f',\n",
    "                'ḟh': 'f', 'bhf': 'f', 'mm': 'm', 'll': 'l', 'nn': 'n',\n",
    "                'ph': 'p', 'bp': 'p', 'rr': 'r', 'sh': 's', 'ṡ': 's',\n",
    "                'th': 't', 'dt': 't', 'he': 'e', 'hé': 'é', 'ha': 'a',\n",
    "                'há': 'á', 'hi': 'i', 'hí': 'í', 'ho': 'o', 'hó': 'ó',\n",
    "                'hu': 'u', 'hú': 'ú', 'n-': '', 'h-': '', 'ss': 's',\n",
    "                'ts': 's', 'n': 'e', 'né': 'é', 'na': 'a', 'ná': 'á',\n",
    "                'ni': 'i', 'ní': 'í', 'no': 'o', 'nó': 'ó', 'nu': 'u',\n",
    "                'nú': 'ú', 'm-': '', 't-': '', 't\\'': ' ', 'm\\'': '',\n",
    "                'd\\'': '', 'l-': '', 'mh': 'm', 'r-': '', 's-': '',\n",
    "                'cc': 'c', 'mh\\'': '', 'g-': 'g'}\n",
    "\n",
    "\n",
    "with open('./data/composite_names.txt', 'r', encoding='utf-8') as f:\n",
    "    names = f.read().split('\\n')\n",
    "    namedict = {name:'_'.join(name.split()) for name in names}\n",
    "    \n",
    "with open('./data/celt_links.txt', 'r', encoding='utf-8') as f:\n",
    "    lst = f.read().split('\\n')\n",
    "    links = {entry.split('\\t')[0]:entry.split('\\t')[1] for entry in lst if len(entry) !=0}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "sfqA7e-IpbuL",
    "outputId": "1621d258-b25f-4ccc-8b8e-8f69bb7ae527"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "485\n"
     ]
    }
   ],
   "source": [
    "print(len(links))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "id": "HRGCUnfypbuS"
   },
   "outputs": [],
   "source": [
    "genres = {}\n",
    "subgenres = {}\n",
    "forms = {}\n",
    "\n",
    "sorted_links = sorted(links.keys())\n",
    "verse = [sorted_links[24]] + sorted_links[195:459] + [sorted_links[462]] + [sorted_links[464]]\n",
    "tracts = sorted_links[459:462] + [sorted_links[463]] + sorted_links[466:]\n",
    "annals = sorted_links[:24] + [sorted_links[25]]\n",
    "\n",
    "for link in verse:\n",
    "    forms[link] = 'Verse'\n",
    "    \n",
    "for link in tracts:\n",
    "    forms[link] = 'Prose'\n",
    "    genres[link] = 'Tract'\n",
    "    \n",
    "for link in annals:\n",
    "    forms[link] = 'Prose'\n",
    "    genres[link] = 'Annals'\n",
    "    subgenres[link] = 'History'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "id": "A3qUeUPfpbuU"
   },
   "outputs": [],
   "source": [
    "def clean(text):\n",
    "    \"\"\"\n",
    "    Cleans text from all sorts of rubbish.\n",
    "    \"\"\"\n",
    "    text = re.sub(r\"\\nU\\d+ \", r\" \", text) # removes error codes\n",
    "    text = re.sub(r\"(p|P)(age)?\\.? ?\\d+\", r\" \", text)  # removes page numbers\n",
    "    text = re.sub(r\"\\{(.+?)\\}\\s*?\", r\" \", text)  # removes MS folios\n",
    "    text = re.sub(r\"(\\.|[a-z])\\d*(\\n|\\s)\", r\"\\1\\2\", text)  # removes footnote numbers\n",
    "    text = re.sub(r\"\\n(\\s*?)¶*(\\d)+~*\\]\", r\"\\n\", text)  # removes line numbers\n",
    "    text = re.sub(r\"[A-Z]{0,4}\\d+[A-Z]{0,3}\\.?( |\\n)\", r\" \", text) # removes random numbers, text ids and sections\n",
    "    text = re.sub(r\"([A-Z]+\\d+)?|Kl\\.?(\\n| )|\", \"\", text) # removes section numbers in annals\n",
    "    text= re.sub(r\" \\d+ \", \" \", text) #removes numbers\n",
    "    text = re.sub(r\"\\.i\\.\", r\"id_est\", text) # changes .i. to 'id_est'\n",
    "    text = re.sub(r\"\\.l\\.\", r\"vel\", text) # changes .l. to 'vel'\n",
    "    text = re.sub(r\" (&|s)?rl\\.?\", r\" etc\", text) # changes &rl to 'etc'\n",
    "    text = re.sub(r\"(\\[.*?\\])\", \"\", text) # removes restored letters and comments in []\n",
    "    text = re.sub(r\"\\n[XVICDML]{1,6}\\.\", \"\", text) # removes roman numbers\n",
    "    text = re.sub(r\"(Text\\s.*?\\n)|((\\.|\\s)[ivxUuírcfl]+?\\.)|sic\\.|(Incipit|Finit|Finis|FINIT|FINIS|Amen|anno\\sdomini|dixit|Author:?|Unknown|folio|Folio)[.,\\n\\s]*\", r\"\", text)  # removes Latin & editor's comments\n",
    "    text = re.sub(r\"(\\w|[.:;,-?!’])\\s([.:;,-?!’])\", r\"\\1\\2\", text) # removes whitespaces between words and punctuation\n",
    "    text = re.sub(r\"^l|l^\", \"\", text) # removes these weird things\n",
    "    text = re.sub(r\"(\\.|\\.’|’|,)(‘|\\w)\", r\"\\1 \\2\", text)  # adds necessary whitespaces\n",
    "    text = re.sub(r\"([.?!]’*)(\\s|‘)\", r\"\\1\\n\\2\", text)  # moves each sentence to a new line\n",
    "    text = re.sub(r\"(\\w|,|&|-|\\s|;)\\n\", r\"\\1 \", text)  # joins sentence parts that were on different lines\n",
    "    text = re.sub(r\"(\\n|\\s|-|‘)\\s+(\\w|&|[’'.:;,-?!]*|\\n)\", r\"\\1\\2\", text)  # removes unnecessary whitespaces\n",
    "    text = re.sub(r\"~|§|\\[|\\]|\\(|\\)|\\|\", r\" \",text)  # removes unnecessary symbols\n",
    "    text = re.sub(r\"\\s-(\\w+)\", r\"-\\1\", text) # removes whitespaces before dashes\n",
    "    text = re.sub(r\"(\\w+)-\\s\", r\"\\1-\", text) # removes whitespaces after dashes\n",
    "    text = re.sub(r\"\\t\", \" \", text) # changes tabs to spaces\n",
    "    text = re.sub(r\"[.,]\", r\" \", text) # removes random stops and commas\n",
    "    text = re.sub(r\" +\", r\" \", text) # replaces multiple whitespaces with one\n",
    "    text = re.sub(r\"\\n+\", r\"\\n\", text) # replaces multiple newlines with one\n",
    "    text = re.sub(r\" \\n\", r\"\\n\", text) # removes whitespaces before newlines\n",
    "    text = re.sub(r\"\\n \", r\"\\n\", text) # removes whitespaces after newlines\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "id": "-SB3e2XrpbuU"
   },
   "outputs": [],
   "source": [
    "# title = re.compile(r'(<h1>|<title>)(.*?)(</h1>|</title>)')\n",
    "editor = re.compile(r'<h3>File Description</h3>(.*?compiled by )?(.*?)</?p>')\n",
    "author = re.compile(r'<h2>Author:\\s+?(.*?)</h2>', re.DOTALL)\n",
    "words = re.compile(r'(\\d+) (words|gmd)')\n",
    "dates = re.compile(r'Date range: (.*?\\d+(–|-)?\\d+(.*?century)?)')\n",
    "# irish = re.compile(r'Language.*?((Old |Middle |(Early Modern )|Classical | )\\S*?Irish).*?</h5>', re.DOTALL)\n",
    "irish_text = re.compile(r'Language.*?Irish.*?</h5>', re.DOTALL)\n",
    "irish = re.compile(r'Old|Middle|Early Modern|Classical.*?Irish', re.DOTALL)\n",
    "langs = re.compile(r'Language.*?\\[([A-Z]{2})\\]', re.DOTALL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "id": "ebwPgTErx6og"
   },
   "outputs": [],
   "source": [
    "test1 = \"Language: [GA] More than 99% is in Middle Irish and Early Modern Irish.</h5>\"\n",
    "test2 = \"Language: [GA] Most of the later annals are in Old, Middle, and Early Modern Irish with very many Latin words and formulae.</h5>\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "id": "4tyuUfxz4FCh"
   },
   "outputs": [],
   "source": [
    "t = irish_text.search(test2).group(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hGPsoCjh4TVC",
    "outputId": "d954879f-c854-4948-9670-74c8897a4749"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Old', 'Middle', 'Early Modern']"
      ]
     },
     "execution_count": 110,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "irish.findall(t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "id": "u7c5vp7bpbuV"
   },
   "outputs": [],
   "source": [
    "def demutate(word):\n",
    "    \"\"\"\n",
    "    Checks if the word is mutated and restores its original form\n",
    "    :param word: str\n",
    "    :return: str\n",
    "    \"\"\"\n",
    "    if len(word) != 0:\n",
    "        if word[:2] in mutations:\n",
    "            word = mutations[word[:2]] + word[2:]\n",
    "        elif word[0] in mutations:\n",
    "            word = mutations[word[0]] + word[1:]\n",
    "    return word\n",
    "\n",
    "\n",
    "def join_names(text):\n",
    "    \"\"\"\n",
    "    Joins multiword names with underscores.\n",
    "    \"\"\"\n",
    "    for k, v in namedict.items():\n",
    "        text = re.sub(k, v, text)\n",
    "    return text\n",
    "\n",
    "def regex_wrapper(regex, gr):\n",
    "    if regex == None:\n",
    "        return \"–\"\n",
    "    else:\n",
    "        return regex.group(gr)\n",
    "\n",
    "def find_info(text):\n",
    "    \"\"\"\n",
    "    Finds metadata on the text's webpage.\n",
    "    \"\"\"\n",
    "    ed = editor.search(text).group(2).replace(\"\\n\", \" \").strip(' ')\n",
    "    au = author.search(text).group(1).replace(\"\\n\", \" \").strip('[] ').capitalize()\n",
    "    wo = words.search(text).group(1).replace(\"\\n\", \"\").strip(' ')\n",
    "    da = regex_wrapper(dates.search(text), 1).replace(\"\\n\", \" \").strip(' ')\n",
    "    ir_text = irish_text.search(text).group(0)\n",
    "    ir = ', '.join(list(set(irish.findall(ir_text))))\n",
    "    # ir = irish.search(irish_text).group(1).replace(\"\\n\", \" \").strip(' ')\n",
    "    la = ', '.join(list(set(langs.findall(text))))\n",
    "    return ed, au, wo, da, ir, la\n",
    "\n",
    "\n",
    "def preprocess_and_count(text):\n",
    "    \"\"\"\n",
    "    Preprocesses text and computes \n",
    "    the number of tokens and types. \n",
    "    NB! Naive tokenization!\n",
    "    \"\"\"\n",
    "    text = re.sub(r'[!\"#$%&()\\*+,\\.:;<=>?@\\[\\]^_`{|}‘’~„“«»†*—]', ' ', text)\n",
    "    text = re.sub(r'((\\r?\\n)+)|( +)', ' ', text)\n",
    "    text = join_names(text)\n",
    "    text = text.lower()\n",
    "    tokens = len(text.split())\n",
    "    unique = len(set(text.split()))    \n",
    "    return text, tokens, unique\n",
    "\n",
    "    \n",
    "def download_page(link):\n",
    "    \"\"\"\n",
    "    Downloads a webpage and parses it.\n",
    "    \"\"\"\n",
    "    try:\n",
    "        source_code = requests.get(link)\n",
    "        plain_text = source_code.text\n",
    "        soup = BeautifulSoup(plain_text, \"html.parser\")\n",
    "        fname = link.split(\"/\")[-1]\n",
    "        # with open(\"./data/raw_celt/\" + fname, \"w\", encoding=\"utf-8\") as f:\n",
    "        #     f.write(plain_text)\n",
    "        return soup\n",
    "    except (ConnectionError, MaxRetryError, NewConnectionError, TimeoutError) as e:\n",
    "        print(\"Connection Error\")\n",
    "        return None\n",
    "\n",
    "\n",
    "def get_text(soup):\n",
    "    \"\"\"\n",
    "    Extracts text itself from the parsed webpage.\n",
    "    \"\"\"\n",
    "    raw_text = soup.get_text()\n",
    "    raw_text = raw_text.split(\"Corpus of Electronic Texts Edition: \")[-1]\n",
    "    return raw_text\n",
    "\n",
    "\n",
    "def get_params(name, link):\n",
    "    \"\"\"\n",
    "    Gets parameters for a single entry.\n",
    "    \"\"\"\n",
    "    page = download_page(link)\n",
    "    if page != None and page.title.text != '404 Not Found':\n",
    "        try:\n",
    "            raw_text = get_text(page)\n",
    "            text = clean(raw_text)\n",
    "            title = name\n",
    "            editor, author, words, dates, irish, langs = find_info(str(page))\n",
    "            preprocessed, tokens, unique = preprocess_and_count(text)\n",
    "            form = forms[link] if link in forms else '–'\n",
    "            genre = genres[link] if link in genres else '–'\n",
    "            subgenre = subgenres[link] if link in subgenres else '–'\n",
    "            params = (title, editor, words, tokens, unique, dates, irish, \n",
    "                    langs, form, genre, subgenre, author, link)\n",
    "            return (params, text, raw_text, preprocessed)\n",
    "        except RecursionError:\n",
    "            print(\"Recursion error: %s\" % link)\n",
    "    else:\n",
    "        print(\"Bad link: %s\" % link)\n",
    "        return None\n",
    "\n",
    "    \n",
    "def add_entry(df, params, pattern):\n",
    "    \"\"\"\n",
    "    Adds a row to the dataset.\n",
    "    \"\"\"\n",
    "    line = eval(pattern % params)\n",
    "    df = df.append(line, ignore_index=True)\n",
    "    return df\n",
    "    \n",
    "\n",
    "def crawl(links):\n",
    "    \"\"\"\n",
    "    Downloads web pages from the list, processes them\n",
    "    and creates a dataset with raw texts, cleaned texts\n",
    "    and metadata.\n",
    "    \"\"\"\n",
    "    data = pd.DataFrame(columns=columns)\n",
    "    for k, v in links.items():\n",
    "        print(k)\n",
    "        res = get_params(v, k)\n",
    "        if res:\n",
    "            data = add_entry(data, res[0], entry_pattern)\n",
    "            data.at[data[data['Title']==res[0][0]].index, \"Raw Text\"] = res[2]\n",
    "            data.at[data[data['Title']==res[0][0]].index, \"Text\"] = res[1]\n",
    "            data.at[data[data['Title']==res[0][0]].index, \"Preprocessed Text\"] = res[3]\n",
    "#           print(res[3][:200])\n",
    "        else:\n",
    "            print(\"No result\")\n",
    "#         time.sleep(2)\n",
    "    return data\n",
    "\n",
    "\n",
    "def save_data(data, filename):\n",
    "    \"\"\"\n",
    "    Saves dataset in both csv and pickle\n",
    "    \"\"\"\n",
    "    data.to_csv(filename+'.csv', encoding='utf-8', sep='\\t', index=False)\n",
    "    with open(filename+'.pkl', 'wb') as f:\n",
    "        pkl.dump(data, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "fb16S3bnpbuW",
    "outputId": "b0c9a8c6-9882-4c3f-ea73-371c751f2a98"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://celt.ucc.ie//published/G100001A.html\n",
      "https://celt.ucc.ie//published/G100001B.html\n",
      "https://celt.ucc.ie//published/G100002.html\n",
      "https://celt.ucc.ie//published/G100004.html\n",
      "https://celt.ucc.ie//published/G100004P.html\n",
      "https://celt.ucc.ie//published/G100005A.html\n",
      "Recursion error: https://celt.ucc.ie//published/G100005A.html\n",
      "No result\n",
      "https://celt.ucc.ie//published/G100005B.html\n",
      "https://celt.ucc.ie//published/G100005C.html\n",
      "https://celt.ucc.ie//published/G100005D.html\n",
      "https://celt.ucc.ie//published/G100005E.html\n",
      "https://celt.ucc.ie//published/G100005F.html\n",
      "https://celt.ucc.ie//published/G100010A.html\n",
      "https://celt.ucc.ie//published/G100011.html\n",
      "https://celt.ucc.ie//published/G100013.html\n",
      "https://celt.ucc.ie//published/G100014.html\n",
      "https://celt.ucc.ie//published/G100015.html\n",
      "https://celt.ucc.ie//published/G100016.html\n",
      "Recursion error: https://celt.ucc.ie//published/G100016.html\n",
      "No result\n",
      "https://celt.ucc.ie//published/G100017.html\n",
      "https://celt.ucc.ie//published/G100018.html\n",
      "https://celt.ucc.ie//published/G100019.html\n",
      "https://celt.ucc.ie//published/G100020.html\n",
      "https://celt.ucc.ie//published/G100021.html\n",
      "https://celt.ucc.ie//published/G100022.html\n",
      "https://celt.ucc.ie//published/G100022A.html\n",
      "https://celt.ucc.ie//published/G100026.html\n",
      "https://celt.ucc.ie//published/G100027.html\n",
      "https://celt.ucc.ie//published/G100028.html\n",
      "https://celt.ucc.ie//published/G100029.html\n",
      "https://celt.ucc.ie//published/G100030.html\n",
      "https://celt.ucc.ie//published/G100032.html\n",
      "https://celt.ucc.ie//published/G100033.html\n",
      "https://celt.ucc.ie//published/G100035.html\n",
      "https://celt.ucc.ie//published/G100036.html\n",
      "https://celt.ucc.ie//published/G100037.html\n",
      "https://celt.ucc.ie//published/G100039.html\n",
      "https://celt.ucc.ie//published/G100040.html\n",
      "https://celt.ucc.ie//published/G100041.html\n",
      "https://celt.ucc.ie//published/G100042.html\n",
      "https://celt.ucc.ie//published/G100043.html\n",
      "https://celt.ucc.ie//published/G100044.html\n",
      "https://celt.ucc.ie//published/G100044A.html\n",
      "https://celt.ucc.ie//published/G100045.html\n",
      "https://celt.ucc.ie//published/G100046.html\n",
      "https://celt.ucc.ie//published/G100047.html\n",
      "https://celt.ucc.ie//published/G100048.html\n",
      "https://celt.ucc.ie//published/G100049.html\n",
      "https://celt.ucc.ie//published/G100050.html\n",
      "https://celt.ucc.ie//published/G100051.html\n",
      "https://celt.ucc.ie//published/G100052.html\n",
      "https://celt.ucc.ie//published/G100053.html\n",
      "https://celt.ucc.ie//published/G100054.html\n",
      "https://celt.ucc.ie//published/G100061.html\n",
      "https://celt.ucc.ie//published/G100062.html\n",
      "https://celt.ucc.ie//published/G100063.html\n",
      "https://celt.ucc.ie//published/G100070.html\n",
      "https://celt.ucc.ie//published/G100080.html\n",
      "https://celt.ucc.ie//published/G102002.html\n",
      "https://celt.ucc.ie//published/G102003.html\n",
      "https://celt.ucc.ie//published/G102004.html\n",
      "https://celt.ucc.ie//published/G102007.html\n",
      "https://celt.ucc.ie//published/G102030.html\n",
      "https://celt.ucc.ie//published/G102900.html\n",
      "https://celt.ucc.ie//published/G105003.html\n",
      "https://celt.ucc.ie//published/G105005.html\n",
      "https://celt.ucc.ie//published/G105007.html\n",
      "https://celt.ucc.ie//published/G105013.html\n",
      "https://celt.ucc.ie//published/G105015.html\n",
      "https://celt.ucc.ie//published/G105016.html\n",
      "https://celt.ucc.ie//published/G105018.html\n",
      "https://celt.ucc.ie//published/G105019.html\n",
      "https://celt.ucc.ie//published/G106001.html\n",
      "https://celt.ucc.ie//published/G106500A.html\n",
      "https://celt.ucc.ie//published/G106500B.html\n",
      "https://celt.ucc.ie//published/G106500C.html\n",
      "https://celt.ucc.ie//published/G106500D.html\n",
      "https://celt.ucc.ie//published/G109001.html\n",
      "https://celt.ucc.ie//published/G200001.html\n",
      "https://celt.ucc.ie//published/G201000.html\n",
      "https://celt.ucc.ie//published/G201001.html\n",
      "https://celt.ucc.ie//published/G201002.html\n",
      "https://celt.ucc.ie//published/G201003.html\n",
      "https://celt.ucc.ie//published/G201005.html\n",
      "https://celt.ucc.ie//published/G201008.html\n",
      "https://celt.ucc.ie//published/G201009.html\n",
      "https://celt.ucc.ie//published/G201010.html\n",
      "https://celt.ucc.ie//published/G201011.html\n",
      "https://celt.ucc.ie//published/G201012.html\n",
      "https://celt.ucc.ie//published/G201013.html\n",
      "https://celt.ucc.ie//published/G201015.html\n",
      "https://celt.ucc.ie//published/G201016.html\n",
      "https://celt.ucc.ie//published/G201018.html\n",
      "https://celt.ucc.ie//published/G201019.html\n",
      "https://celt.ucc.ie//published/G201020.html\n",
      "https://celt.ucc.ie//published/G201023.html\n",
      "https://celt.ucc.ie//published/G201025.html\n",
      "https://celt.ucc.ie//published/G201027.html\n",
      "https://celt.ucc.ie//published/G201028.html\n",
      "https://celt.ucc.ie//published/G201029.html\n",
      "https://celt.ucc.ie//published/G201030.html\n",
      "https://celt.ucc.ie//published/G201031.html\n",
      "https://celt.ucc.ie//published/G201032.html\n",
      "https://celt.ucc.ie//published/G201033.html\n",
      "https://celt.ucc.ie//published/G201036.html\n",
      "https://celt.ucc.ie//published/G201040.html\n",
      "https://celt.ucc.ie//published/G202001.html\n",
      "https://celt.ucc.ie//published/G207000.html\n",
      "https://celt.ucc.ie//published/G207001.html\n",
      "https://celt.ucc.ie//published/G207002.html\n",
      "https://celt.ucc.ie//published/G207003.html\n",
      "https://celt.ucc.ie//published/G207005.html\n",
      "https://celt.ucc.ie//published/G207006.html\n",
      "https://celt.ucc.ie//published/G207007.html\n",
      "https://celt.ucc.ie//published/G207008.html\n",
      "https://celt.ucc.ie//published/G207009A.html\n",
      "https://celt.ucc.ie//published/G300001.html\n",
      "https://celt.ucc.ie//published/G300003.html\n",
      "https://celt.ucc.ie//published/G300004.html\n",
      "https://celt.ucc.ie//published/G300005.html\n",
      "https://celt.ucc.ie//published/G300006.html\n",
      "https://celt.ucc.ie//published/G300007.html\n",
      "https://celt.ucc.ie//published/G300008.html\n",
      "https://celt.ucc.ie//published/G300009.html\n",
      "https://celt.ucc.ie//published/G300010.html\n",
      "https://celt.ucc.ie//published/G300011.html\n",
      "https://celt.ucc.ie//published/G300012.html\n",
      "https://celt.ucc.ie//published/G301000.html\n",
      "https://celt.ucc.ie//published/G301005.html\n",
      "https://celt.ucc.ie//published/G301006.html\n",
      "https://celt.ucc.ie//published/G301008.html\n",
      "https://celt.ucc.ie//published/G301010.html\n",
      "https://celt.ucc.ie//published/G301012.html\n",
      "https://celt.ucc.ie//published/G301013.html\n",
      "https://celt.ucc.ie//published/G301015.html\n",
      "https://celt.ucc.ie//published/G301016.html\n",
      "https://celt.ucc.ie//published/G301017.html\n",
      "https://celt.ucc.ie//published/G301018.html\n",
      "https://celt.ucc.ie//published/G301020.html\n",
      "https://celt.ucc.ie//published/G301021.html\n",
      "https://celt.ucc.ie//published/G301021A.html\n",
      "https://celt.ucc.ie//published/G301022.html\n",
      "https://celt.ucc.ie//published/G301023.html\n",
      "https://celt.ucc.ie//published/G301024.html\n",
      "https://celt.ucc.ie//published/G301025.html\n",
      "https://celt.ucc.ie//published/G301029.html\n",
      "https://celt.ucc.ie//published/G301030.html\n",
      "https://celt.ucc.ie//published/G301031.html\n",
      "https://celt.ucc.ie//published/G301032.html\n",
      "https://celt.ucc.ie//published/G301033.html\n",
      "https://celt.ucc.ie//published/G301035.html\n",
      "https://celt.ucc.ie//published/G301037.html\n",
      "https://celt.ucc.ie//published/G301038.html\n",
      "https://celt.ucc.ie//published/G301040.html\n",
      "https://celt.ucc.ie//published/G301041.html\n",
      "https://celt.ucc.ie//published/G301042.html\n",
      "https://celt.ucc.ie//published/G301043.html\n",
      "https://celt.ucc.ie//published/G301044.html\n",
      "https://celt.ucc.ie//published/G301045.html\n",
      "https://celt.ucc.ie//published/G302001.html\n",
      "https://celt.ucc.ie//published/G302007.html\n",
      "https://celt.ucc.ie//published/G302010.html\n",
      "https://celt.ucc.ie//published/G302011.html\n",
      "https://celt.ucc.ie//published/G302012.html\n",
      "https://celt.ucc.ie//published/G302013.html\n",
      "https://celt.ucc.ie//published/G302014.html\n",
      "https://celt.ucc.ie//published/G302016.html\n",
      "https://celt.ucc.ie//published/G302017.html\n",
      "https://celt.ucc.ie//published/G302018.html\n",
      "https://celt.ucc.ie//published/G302019.html\n",
      "https://celt.ucc.ie//published/G302020.html\n",
      "https://celt.ucc.ie//published/G302022.html\n",
      "https://celt.ucc.ie//published/G302025.html\n",
      "https://celt.ucc.ie//published/G302026.html\n",
      "https://celt.ucc.ie//published/G303000.html\n",
      "https://celt.ucc.ie//published/G303002.html\n",
      "https://celt.ucc.ie//published/G303003.html\n",
      "https://celt.ucc.ie//published/G303005.html\n",
      "https://celt.ucc.ie//published/G303006.html\n",
      "https://celt.ucc.ie//published/G303009.html\n",
      "https://celt.ucc.ie//published/G303010.html\n",
      "https://celt.ucc.ie//published/G303011.html\n",
      "https://celt.ucc.ie//published/G303012.html\n",
      "https://celt.ucc.ie//published/G303013.html\n",
      "https://celt.ucc.ie//published/G303014.html\n",
      "https://celt.ucc.ie//published/G303015.html\n",
      "https://celt.ucc.ie//published/G303016.html\n",
      "https://celt.ucc.ie//published/G303017.html\n",
      "https://celt.ucc.ie//published/G303018.html\n",
      "https://celt.ucc.ie//published/G303027.html\n",
      "https://celt.ucc.ie//published/G303028.html\n",
      "https://celt.ucc.ie//published/G303029.html\n",
      "https://celt.ucc.ie//published/G303030.html\n",
      "https://celt.ucc.ie//published/G305000.html\n",
      "https://celt.ucc.ie//published/G305002.html\n",
      "https://celt.ucc.ie//published/G308002.html\n",
      "https://celt.ucc.ie//published/G400001.html\n",
      "https://celt.ucc.ie//published/G400002.html\n",
      "https://celt.ucc.ie//published/G400003.html\n",
      "https://celt.ucc.ie//published/G400004.html\n",
      "https://celt.ucc.ie//published/G400007.html\n",
      "https://celt.ucc.ie//published/G400008.html\n",
      "https://celt.ucc.ie//published/G400009.html\n",
      "https://celt.ucc.ie//published/G400011.html\n",
      "https://celt.ucc.ie//published/G400012.html\n",
      "https://celt.ucc.ie//published/G400013.html\n",
      "https://celt.ucc.ie//published/G400014.html\n",
      "https://celt.ucc.ie//published/G400015.html\n",
      "https://celt.ucc.ie//published/G400016.html\n",
      "https://celt.ucc.ie//published/G400018.html\n",
      "https://celt.ucc.ie//published/G400019.html\n",
      "https://celt.ucc.ie//published/G400021.html\n",
      "https://celt.ucc.ie//published/G400023.html\n",
      "https://celt.ucc.ie//published/G400024.html\n",
      "https://celt.ucc.ie//published/G400025.html\n",
      "https://celt.ucc.ie//published/G400026.html\n",
      "https://celt.ucc.ie//published/G400027.html\n",
      "https://celt.ucc.ie//published/G400028.html\n",
      "https://celt.ucc.ie//published/G400030.html\n",
      "https://celt.ucc.ie//published/G400032.html\n",
      "https://celt.ucc.ie//published/G400038.html\n",
      "https://celt.ucc.ie//published/G400043.html\n",
      "https://celt.ucc.ie//published/G400044.html\n",
      "https://celt.ucc.ie//published/G400045.html\n",
      "https://celt.ucc.ie//published/G400046.html\n",
      "https://celt.ucc.ie//published/G400047.html\n",
      "https://celt.ucc.ie//published/G400050.html\n",
      "https://celt.ucc.ie//published/G400052.html\n",
      "https://celt.ucc.ie//published/G400053.html\n",
      "https://celt.ucc.ie//published/G400055.html\n",
      "https://celt.ucc.ie//published/G400058.html\n",
      "https://celt.ucc.ie//published/G400110.html\n",
      "https://celt.ucc.ie//published/G402015.html\n",
      "https://celt.ucc.ie//published/G402020.html\n",
      "https://celt.ucc.ie//published/G402021.html\n",
      "https://celt.ucc.ie//published/G402022.html\n",
      "https://celt.ucc.ie//published/G402023.html\n",
      "https://celt.ucc.ie//published/G402024.html\n",
      "https://celt.ucc.ie//published/G402025.html\n",
      "https://celt.ucc.ie//published/G402026.html\n",
      "https://celt.ucc.ie//published/G402027.html\n",
      "https://celt.ucc.ie//published/G402028.html\n",
      "https://celt.ucc.ie//published/G402029.html\n",
      "https://celt.ucc.ie//published/G402030.html\n",
      "https://celt.ucc.ie//published/G402031.html\n",
      "https://celt.ucc.ie//published/G402032.html\n",
      "https://celt.ucc.ie//published/G402033.html\n",
      "https://celt.ucc.ie//published/G402034.html\n",
      "https://celt.ucc.ie//published/G402035.html\n",
      "https://celt.ucc.ie//published/G402037.html\n",
      "https://celt.ucc.ie//published/G402038.html\n",
      "https://celt.ucc.ie//published/G402039.html\n",
      "https://celt.ucc.ie//published/G402040.html\n",
      "https://celt.ucc.ie//published/G402041.html\n",
      "https://celt.ucc.ie//published/G402042.html\n",
      "https://celt.ucc.ie//published/G402043.html\n",
      "https://celt.ucc.ie//published/G402044.html\n",
      "https://celt.ucc.ie//published/G402045.html\n",
      "https://celt.ucc.ie//published/G402046.html\n",
      "https://celt.ucc.ie//published/G402047.html\n",
      "https://celt.ucc.ie//published/G402048.html\n",
      "https://celt.ucc.ie//published/G402049.html\n",
      "https://celt.ucc.ie//published/G402051.html\n",
      "https://celt.ucc.ie//published/G402052.html\n",
      "https://celt.ucc.ie//published/G402053.html\n",
      "https://celt.ucc.ie//published/G402054.html\n",
      "https://celt.ucc.ie//published/G402055.html\n",
      "https://celt.ucc.ie//published/G402056.html\n",
      "https://celt.ucc.ie//published/G402057.html\n",
      "https://celt.ucc.ie//published/G402058.html\n",
      "https://celt.ucc.ie//published/G402059.html\n",
      "https://celt.ucc.ie//published/G402060.html\n",
      "https://celt.ucc.ie//published/G402061.html\n",
      "https://celt.ucc.ie//published/G402062.html\n",
      "https://celt.ucc.ie//published/G402063.html\n",
      "https://celt.ucc.ie//published/G402064.html\n",
      "https://celt.ucc.ie//published/G402065.html\n",
      "https://celt.ucc.ie//published/G402066.html\n",
      "https://celt.ucc.ie//published/G402067.html\n",
      "https://celt.ucc.ie//published/G402068.html\n",
      "https://celt.ucc.ie//published/G402069.html\n",
      "https://celt.ucc.ie//published/G402070.html\n",
      "https://celt.ucc.ie//published/G402071.html\n",
      "https://celt.ucc.ie//published/G402072.html\n",
      "https://celt.ucc.ie//published/G402073.html\n",
      "https://celt.ucc.ie//published/G402074.html\n",
      "https://celt.ucc.ie//published/G402075.html\n",
      "https://celt.ucc.ie//published/G402076.html\n",
      "https://celt.ucc.ie//published/G402077.html\n",
      "https://celt.ucc.ie//published/G402078.html\n",
      "https://celt.ucc.ie//published/G402079.html\n",
      "https://celt.ucc.ie//published/G402080.html\n",
      "https://celt.ucc.ie//published/G402081.html\n",
      "https://celt.ucc.ie//published/G402082.html\n",
      "https://celt.ucc.ie//published/G402083.html\n",
      "https://celt.ucc.ie//published/G402084.html\n",
      "https://celt.ucc.ie//published/G402085.html\n",
      "https://celt.ucc.ie//published/G402086.html\n",
      "https://celt.ucc.ie//published/G402087.html\n",
      "https://celt.ucc.ie//published/G402088.html\n",
      "https://celt.ucc.ie//published/G402089.html\n",
      "https://celt.ucc.ie//published/G402090.html\n",
      "https://celt.ucc.ie//published/G402091.html\n",
      "https://celt.ucc.ie//published/G402092.html\n",
      "https://celt.ucc.ie//published/G402093.html\n",
      "https://celt.ucc.ie//published/G402094.html\n",
      "https://celt.ucc.ie//published/G402095.html\n",
      "https://celt.ucc.ie//published/G402096.html\n",
      "https://celt.ucc.ie//published/G402097.html\n",
      "https://celt.ucc.ie//published/G402098.html\n",
      "https://celt.ucc.ie//published/G402099.html\n",
      "https://celt.ucc.ie//published/G402100.html\n",
      "https://celt.ucc.ie//published/G402101.html\n",
      "https://celt.ucc.ie//published/G402102.html\n",
      "https://celt.ucc.ie//published/G402103.html\n",
      "https://celt.ucc.ie//published/G402104.html\n",
      "https://celt.ucc.ie//published/G402105.html\n",
      "https://celt.ucc.ie//published/G402107.html\n",
      "https://celt.ucc.ie//published/G402108.html\n",
      "https://celt.ucc.ie//published/G402109.html\n",
      "https://celt.ucc.ie//published/G402110.html\n",
      "https://celt.ucc.ie//published/G402111.html\n",
      "https://celt.ucc.ie//published/G402112.html\n",
      "https://celt.ucc.ie//published/G402114.html\n",
      "https://celt.ucc.ie//published/G402115.html\n",
      "https://celt.ucc.ie//published/G402116.html\n",
      "https://celt.ucc.ie//published/G402117.html\n",
      "https://celt.ucc.ie//published/G402118.html\n",
      "https://celt.ucc.ie//published/G402119.html\n",
      "https://celt.ucc.ie//published/G402120.html\n",
      "https://celt.ucc.ie//published/G402121.html\n",
      "https://celt.ucc.ie//published/G402122.html\n",
      "https://celt.ucc.ie//published/G402123.html\n",
      "https://celt.ucc.ie//published/G402124.html\n",
      "https://celt.ucc.ie//published/G402125.html\n",
      "https://celt.ucc.ie//published/G402126.html\n",
      "https://celt.ucc.ie//published/G402127.html\n",
      "https://celt.ucc.ie//published/G402128.html\n",
      "https://celt.ucc.ie//published/G402129.html\n",
      "https://celt.ucc.ie//published/G402130.html\n",
      "https://celt.ucc.ie//published/G402131.html\n",
      "https://celt.ucc.ie//published/G402132.html\n",
      "https://celt.ucc.ie//published/G402133.html\n",
      "https://celt.ucc.ie//published/G402134.html\n",
      "https://celt.ucc.ie//published/G402135.html\n",
      "https://celt.ucc.ie//published/G402136.html\n",
      "https://celt.ucc.ie//published/G402137.html\n",
      "https://celt.ucc.ie//published/G402138.html\n",
      "https://celt.ucc.ie//published/G402139.html\n",
      "https://celt.ucc.ie//published/G402140.html\n",
      "https://celt.ucc.ie//published/G402141.html\n",
      "https://celt.ucc.ie//published/G402142.html\n",
      "https://celt.ucc.ie//published/G402143.html\n",
      "https://celt.ucc.ie//published/G402144.html\n",
      "https://celt.ucc.ie//published/G402145.html\n",
      "https://celt.ucc.ie//published/G402208.html\n",
      "https://celt.ucc.ie//published/G402209.html\n",
      "https://celt.ucc.ie//published/G402210.html\n",
      "https://celt.ucc.ie//published/G402211.html\n",
      "https://celt.ucc.ie//published/G402212.html\n",
      "https://celt.ucc.ie//published/G402213.html\n",
      "https://celt.ucc.ie//published/G402214.html\n",
      "https://celt.ucc.ie//published/G402215.html\n",
      "https://celt.ucc.ie//published/G402216.html\n",
      "https://celt.ucc.ie//published/G402217.html\n",
      "https://celt.ucc.ie//published/G402218.html\n",
      "https://celt.ucc.ie//published/G402219.html\n",
      "https://celt.ucc.ie//published/G402220.html\n",
      "https://celt.ucc.ie//published/G402221.html\n",
      "https://celt.ucc.ie//published/G402222.html\n",
      "https://celt.ucc.ie//published/G402223.html\n",
      "https://celt.ucc.ie//published/G402224.html\n",
      "https://celt.ucc.ie//published/G402225.html\n",
      "https://celt.ucc.ie//published/G402226.html\n",
      "https://celt.ucc.ie//published/G402227.html\n",
      "https://celt.ucc.ie//published/G402228.html\n",
      "https://celt.ucc.ie//published/G402229.html\n",
      "https://celt.ucc.ie//published/G402230.html\n",
      "https://celt.ucc.ie//published/G402231.html\n",
      "https://celt.ucc.ie//published/G402232.html\n",
      "https://celt.ucc.ie//published/G402233.html\n",
      "https://celt.ucc.ie//published/G402234.html\n",
      "https://celt.ucc.ie//published/G402235.html\n",
      "https://celt.ucc.ie//published/G402236.html\n",
      "https://celt.ucc.ie//published/G402237.html\n",
      "https://celt.ucc.ie//published/G402238.html\n",
      "https://celt.ucc.ie//published/G402239.html\n",
      "https://celt.ucc.ie//published/G402240.html\n",
      "https://celt.ucc.ie//published/G402241.html\n",
      "https://celt.ucc.ie//published/G402242.html\n",
      "https://celt.ucc.ie//published/G402243.html\n",
      "https://celt.ucc.ie//published/G402244.html\n",
      "https://celt.ucc.ie//published/G402245.html\n",
      "https://celt.ucc.ie//published/G402246.html\n",
      "https://celt.ucc.ie//published/G402247.html\n",
      "https://celt.ucc.ie//published/G402248.html\n",
      "https://celt.ucc.ie//published/G402249.html\n",
      "https://celt.ucc.ie//published/G402250.html\n",
      "https://celt.ucc.ie//published/G402251.html\n",
      "https://celt.ucc.ie//published/G402252.html\n",
      "https://celt.ucc.ie//published/G402253.html\n",
      "https://celt.ucc.ie//published/G402254.html\n",
      "https://celt.ucc.ie//published/G402255.html\n",
      "https://celt.ucc.ie//published/G402258.html\n",
      "https://celt.ucc.ie//published/G402260.html\n",
      "https://celt.ucc.ie//published/G402262.html\n",
      "https://celt.ucc.ie//published/G402263.html\n",
      "https://celt.ucc.ie//published/G402264.html\n",
      "https://celt.ucc.ie//published/G402265.html\n",
      "https://celt.ucc.ie//published/G402267.html\n",
      "https://celt.ucc.ie//published/G402268.html\n",
      "https://celt.ucc.ie//published/G402270.html\n",
      "https://celt.ucc.ie//published/G402271.html\n",
      "https://celt.ucc.ie//published/G402272.html\n",
      "https://celt.ucc.ie//published/G402282.html\n",
      "https://celt.ucc.ie//published/G402284.html\n",
      "https://celt.ucc.ie//published/G402285.html\n",
      "https://celt.ucc.ie//published/G402286.html\n",
      "https://celt.ucc.ie//published/G402288.html\n",
      "https://celt.ucc.ie//published/G402292.html\n",
      "https://celt.ucc.ie//published/G402293.html\n",
      "https://celt.ucc.ie//published/G402294.html\n",
      "https://celt.ucc.ie//published/G402295.html\n",
      "https://celt.ucc.ie//published/G402296.html\n",
      "https://celt.ucc.ie//published/G402298.html\n",
      "https://celt.ucc.ie//published/G402300.html\n",
      "https://celt.ucc.ie//published/G402301.html\n",
      "https://celt.ucc.ie//published/G402302.html\n",
      "https://celt.ucc.ie//published/G402303.html\n",
      "https://celt.ucc.ie//published/G402304.html\n",
      "https://celt.ucc.ie//published/G402305.html\n",
      "https://celt.ucc.ie//published/G402306.html\n",
      "https://celt.ucc.ie//published/G402307.html\n",
      "https://celt.ucc.ie//published/G402308.html\n",
      "https://celt.ucc.ie//published/G402311.html\n",
      "https://celt.ucc.ie//published/G402313.html\n",
      "https://celt.ucc.ie//published/G402314.html\n",
      "https://celt.ucc.ie//published/G402315.html\n",
      "https://celt.ucc.ie//published/G402323.html\n",
      "https://celt.ucc.ie//published/G402324.html\n",
      "https://celt.ucc.ie//published/G402327.html\n",
      "https://celt.ucc.ie//published/G402329.html\n",
      "https://celt.ucc.ie//published/G402333.html\n",
      "https://celt.ucc.ie//published/G402340.html\n",
      "https://celt.ucc.ie//published/G402342.html\n",
      "https://celt.ucc.ie//published/G402344.html\n",
      "https://celt.ucc.ie//published/G402346.html\n",
      "https://celt.ucc.ie//published/G402348.html\n",
      "https://celt.ucc.ie//published/G402350.html\n",
      "https://celt.ucc.ie//published/G402351.html\n",
      "https://celt.ucc.ie//published/G402354.html\n",
      "https://celt.ucc.ie//published/G402365.html\n",
      "https://celt.ucc.ie//published/G402560.html\n",
      "https://celt.ucc.ie//published/G402561.html\n",
      "https://celt.ucc.ie//published/G402562.html\n",
      "https://celt.ucc.ie//published/G402563.html\n",
      "https://celt.ucc.ie//published/G405990.html\n",
      "https://celt.ucc.ie//published/G406000.html\n",
      "https://celt.ucc.ie//published/G500001.html\n",
      "https://celt.ucc.ie//published/G502003.html\n",
      "https://celt.ucc.ie//published/G502006.html\n",
      "https://celt.ucc.ie//published/G502007.html\n",
      "https://celt.ucc.ie//published/G503000.html\n",
      "https://celt.ucc.ie//published/G503002.html\n",
      "https://celt.ucc.ie//published/G503020.html\n",
      "https://celt.ucc.ie//published/G503021.html\n",
      "https://celt.ucc.ie//published/G504001.html\n",
      "https://celt.ucc.ie//published/G505000.html\n",
      "https://celt.ucc.ie//published/G600005.html\n",
      "https://celt.ucc.ie//published/G600008.html\n",
      "https://celt.ucc.ie//published/G600009A.html\n",
      "https://celt.ucc.ie//published/G600009B.html\n",
      "https://celt.ucc.ie//published/G600009C.html\n",
      "https://celt.ucc.ie//published/G600010.html\n",
      "https://celt.ucc.ie//published/G600011.html\n",
      "https://celt.ucc.ie//published/G600012.html\n",
      "https://celt.ucc.ie//published/G600013.html\n",
      "https://celt.ucc.ie//published/G600014.html\n",
      "https://celt.ucc.ie//published/G600015.html\n",
      "https://celt.ucc.ie//published/G600016.html\n",
      "https://celt.ucc.ie//published/G600018.html\n",
      "https://celt.ucc.ie//published/G600019.html\n",
      "https://celt.ucc.ie//published/G600020.html\n",
      "https://celt.ucc.ie//published/G600021.html\n",
      "https://celt.ucc.ie//published/G600030.html\n",
      "https://celt.ucc.ie//published/G602001.html\n",
      "https://celt.ucc.ie//published/G602002.html\n",
      "CPU times: user 1min 44s, sys: 1.14 s, total: 1min 45s\n",
      "Wall time: 5min 56s\n"
     ]
    }
   ],
   "source": [
    "%time data = crawl(links)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "flAzQ3xOpbuX",
    "outputId": "8ed377d2-13e6-4933-db93-108d3b43e2cc"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "483"
      ]
     },
     "execution_count": 122,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 677
    },
    "id": "cn-Ofqh5pbuY",
    "outputId": "ca6e1d2f-c4fe-4165-cd25-5ea92f63c0e0"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Editor</th>\n",
       "      <th>Size (words)</th>\n",
       "      <th>Real size (tokens)</th>\n",
       "      <th>Unique forms</th>\n",
       "      <th>Dates</th>\n",
       "      <th>Irish</th>\n",
       "      <th>Other languages</th>\n",
       "      <th>Form</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Subgenre</th>\n",
       "      <th>Group</th>\n",
       "      <th>Author</th>\n",
       "      <th>Link</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Raw Text</th>\n",
       "      <th>Text</th>\n",
       "      <th>Preprocessed Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The Annals of Ulster (to AD 1201)</td>\n",
       "      <td>Donnchadh Ó Corráin, Mavis Cournane</td>\n",
       "      <td>118200</td>\n",
       "      <td>65982</td>\n",
       "      <td>12514</td>\n",
       "      <td>c. 550</td>\n",
       "      <td></td>\n",
       "      <td>LA, EN, GA</td>\n",
       "      <td>Prose</td>\n",
       "      <td>Annals</td>\n",
       "      <td>History</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G100001A.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G100001A\\nThe Annals of Ulster: Author:\\t[unkn...</td>\n",
       "      <td>The Annals of Ulster: List of witnesses H: Du...</td>\n",
       "      <td>the annals of ulster list of witnesses h dubl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The Annals of Ulster (to AD 1378)</td>\n",
       "      <td>Donnchadh Ó Corráin</td>\n",
       "      <td>760</td>\n",
       "      <td>28984</td>\n",
       "      <td>5329</td>\n",
       "      <td>c. 550</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "      <td>Prose</td>\n",
       "      <td>Annals</td>\n",
       "      <td>History</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G100001B.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G100001B\\nAnnala Uladh: Annals of Ulster other...</td>\n",
       "      <td>Annala Uladh: Annals of Ulster otherwise Anna...</td>\n",
       "      <td>annala uladh annals of ulster otherwise annal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Annals of Tigernach</td>\n",
       "      <td>Donnchadh Ó Corráin</td>\n",
       "      <td>870</td>\n",
       "      <td>42546</td>\n",
       "      <td>9266</td>\n",
       "      <td>Mid-6th century to 1178</td>\n",
       "      <td>Early Modern</td>\n",
       "      <td></td>\n",
       "      <td>Prose</td>\n",
       "      <td>Annals</td>\n",
       "      <td>History</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G100002.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G100002\\nAnnals of Tigernach: Author:\\t[unknow...</td>\n",
       "      <td>Annals of Tigernach: K\\nui Quies sancti Ciann...</td>\n",
       "      <td>annals of tigernach k ui quies sancti ciannai...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Annals of Inisfallen</td>\n",
       "      <td>Seán Mac Airt</td>\n",
       "      <td>65500</td>\n",
       "      <td>35811</td>\n",
       "      <td>8868</td>\n",
       "      <td>550-1326</td>\n",
       "      <td>Old, Middle, Early Modern</td>\n",
       "      <td>LA, EN, GA</td>\n",
       "      <td>Prose</td>\n",
       "      <td>Annals</td>\n",
       "      <td>History</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G100004.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G100004\\nAnnals of Inisfallen: Author:\\tunknow...</td>\n",
       "      <td>Annals of Inisfallen: unknown Prima feria\\nCo...</td>\n",
       "      <td>annals of inisfallen unknown prima feria conu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Annals of Inisfallen, Pre-Patrician section</td>\n",
       "      <td></td>\n",
       "      <td>12450</td>\n",
       "      <td>7342</td>\n",
       "      <td>3199</td>\n",
       "      <td>AD 550-1326</td>\n",
       "      <td>Middle</td>\n",
       "      <td>LA, EN, GA</td>\n",
       "      <td>Prose</td>\n",
       "      <td>Annals</td>\n",
       "      <td>History</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G100004P.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G100004P\\nAnnals of Inisfallen, Pre-Patrician ...</td>\n",
       "      <td>Annals of Inisfallen Pre-Patrician Section: u...</td>\n",
       "      <td>annals of inisfallen pre-patrician section un...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         Title  ...                                  Preprocessed Text\n",
       "0            The Annals of Ulster (to AD 1201)  ...   the annals of ulster list of witnesses h dubl...\n",
       "1            The Annals of Ulster (to AD 1378)  ...   annala uladh annals of ulster otherwise annal...\n",
       "2                          Annals of Tigernach  ...   annals of tigernach k ui quies sancti ciannai...\n",
       "3                         Annals of Inisfallen  ...   annals of inisfallen unknown prima feria conu...\n",
       "4  Annals of Inisfallen, Pre-Patrician section  ...   annals of inisfallen pre-patrician section un...\n",
       "\n",
       "[5 rows x 18 columns]"
      ]
     },
     "execution_count": 123,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 538
    },
    "id": "1Glc7qp3pbub",
    "outputId": "d2f30f0c-d4fd-4778-b1b1-f6eeb6aa3404",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Title</th>\n",
       "      <th>Editor</th>\n",
       "      <th>Size (words)</th>\n",
       "      <th>Real size (tokens)</th>\n",
       "      <th>Unique forms</th>\n",
       "      <th>Dates</th>\n",
       "      <th>Irish</th>\n",
       "      <th>Other languages</th>\n",
       "      <th>Form</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Subgenre</th>\n",
       "      <th>Group</th>\n",
       "      <th>Author</th>\n",
       "      <th>Link</th>\n",
       "      <th>Comment</th>\n",
       "      <th>Raw Text</th>\n",
       "      <th>Text</th>\n",
       "      <th>Preprocessed Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>478</th>\n",
       "      <td>A Treatise on Fevers</td>\n",
       "      <td>Beatrix Färber</td>\n",
       "      <td>19540</td>\n",
       "      <td>12946</td>\n",
       "      <td>2017</td>\n",
       "      <td>–</td>\n",
       "      <td>Early Modern</td>\n",
       "      <td>LA, AR, GA, EN</td>\n",
       "      <td>Prose</td>\n",
       "      <td>Tract</td>\n",
       "      <td>–</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G600020.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G600020\\nA Treatise on Fevers: Author:\\tUnknow...</td>\n",
       "      <td>A Treatise on Fevers: List of witnesses A: Du...</td>\n",
       "      <td>a treatise on fevers list of witnesses a dubl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>479</th>\n",
       "      <td>De Dosibus medicinarum</td>\n",
       "      <td>Shawn Sheahan</td>\n",
       "      <td>19950</td>\n",
       "      <td>12241</td>\n",
       "      <td>2784</td>\n",
       "      <td>–</td>\n",
       "      <td>Old, Middle</td>\n",
       "      <td>LA, EN, GR, GA</td>\n",
       "      <td>Prose</td>\n",
       "      <td>Tract</td>\n",
       "      <td>–</td>\n",
       "      <td>—</td>\n",
       "      <td>Walter de agilon/galterius agilinus</td>\n",
       "      <td>https://celt.ucc.ie//published/G600021.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G600021\\nAn Irish Version of Gualterus de Dosi...</td>\n",
       "      <td>An Irish Version of Gualterus de Dosibus: Wal...</td>\n",
       "      <td>an irish version of gualterus de dosibus walt...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>480</th>\n",
       "      <td>An Irish Astronomical Tract</td>\n",
       "      <td>Beatrix Färber, Ruth Murphy</td>\n",
       "      <td>98800</td>\n",
       "      <td>79748</td>\n",
       "      <td>6417</td>\n",
       "      <td>c.1300–1350</td>\n",
       "      <td></td>\n",
       "      <td>LA, GA, AR, FR, EN</td>\n",
       "      <td>Prose</td>\n",
       "      <td>Tract</td>\n",
       "      <td>–</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G600030.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G600030\\nAn Irish Astronomical Tract: Author:\\...</td>\n",
       "      <td>An Irish Astronomical Tract: Acknowlegements ...</td>\n",
       "      <td>an irish astronomical tract acknowlegements c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>481</th>\n",
       "      <td>The Meaning of Birthdays</td>\n",
       "      <td>Beatrix Färber</td>\n",
       "      <td>1135</td>\n",
       "      <td>559</td>\n",
       "      <td>254</td>\n",
       "      <td>c.1000–1300</td>\n",
       "      <td>Middle</td>\n",
       "      <td>EN, GA</td>\n",
       "      <td>Prose</td>\n",
       "      <td>Tract</td>\n",
       "      <td>–</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G602001.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G602001\\nThe Meaning of Birthdays: Author:\\t[u...</td>\n",
       "      <td>The Meaning of Birthdays: List of witnesses :...</td>\n",
       "      <td>the meaning of birthdays list of witnesses tr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>482</th>\n",
       "      <td>Ranna an aeir</td>\n",
       "      <td>A. O. Anderson</td>\n",
       "      <td>5650</td>\n",
       "      <td>4411</td>\n",
       "      <td>1382</td>\n",
       "      <td>c.1500–1550</td>\n",
       "      <td>Early Modern</td>\n",
       "      <td>LA, EN, GA</td>\n",
       "      <td>Prose</td>\n",
       "      <td>Tract</td>\n",
       "      <td>–</td>\n",
       "      <td>—</td>\n",
       "      <td>Unknown</td>\n",
       "      <td>https://celt.ucc.ie//published/G602002.html</td>\n",
       "      <td>—</td>\n",
       "      <td>G602002\\nRanna an aeir: Author:\\t[unknown]\\n1[...</td>\n",
       "      <td>Ranna an aeir:1 Fiarfaighthear and so ca lin ...</td>\n",
       "      <td>ranna an aeir 1 fiarfaighthear and so ca lin ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Title  ...                                  Preprocessed Text\n",
       "478         A Treatise on Fevers  ...   a treatise on fevers list of witnesses a dubl...\n",
       "479       De Dosibus medicinarum  ...   an irish version of gualterus de dosibus walt...\n",
       "480  An Irish Astronomical Tract  ...   an irish astronomical tract acknowlegements c...\n",
       "481     The Meaning of Birthdays  ...   the meaning of birthdays list of witnesses tr...\n",
       "482                Ranna an aeir  ...   ranna an aeir 1 fiarfaighthear and so ca lin ...\n",
       "\n",
       "[5 rows x 18 columns]"
      ]
     },
     "execution_count": 124,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "id": "jkpJQITspbud"
   },
   "outputs": [],
   "source": [
    "save_data(data, \"crawled_celt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "CR8JGmkfpbue"
   },
   "outputs": [],
   "source": [
    "for title, text in zip(data['Title'], data['Text']):\n",
    "    with open(\"./crawled_celt/\"+title.strip('?')+\".txt\", \"w\", encoding=\"utf-8\") as f:\n",
    "        f.write(text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RZzX7jhEpbue"
   },
   "source": [
    "## Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "adBYw_yBpbuf"
   },
   "outputs": [],
   "source": [
    "str(download_page('https://celt.ucc.ie//published/G201014.html').title.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "gTwCPkmqpbuf"
   },
   "outputs": [],
   "source": [
    "link = 'https://celt.ucc.ie//published/G402111.html'\n",
    "name = links[link]\n",
    "page = download_page(link)\n",
    "editor, author, words, dates, irish, langs = find_info(str(page))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pNhbU4Vipbug"
   },
   "outputs": [],
   "source": [
    "print(name, editor, author, words, dates, irish, langs, sep=\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GWAkDOWTpbug"
   },
   "outputs": [],
   "source": [
    "raw_text = get_text(page)\n",
    "text = clean(raw_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Z9VRnYF5pbug"
   },
   "outputs": [],
   "source": [
    "raw_text[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "cbm-Q1PMpbuh"
   },
   "outputs": [],
   "source": [
    "text[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wJNZkwcjpbuh"
   },
   "outputs": [],
   "source": [
    "preprocessed, tokens, unique = preprocess_and_count(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "_HxtnWuHpbui"
   },
   "outputs": [],
   "source": [
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pinGLfatpbui"
   },
   "outputs": [],
   "source": [
    "unique"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "hkauX89Qpbui"
   },
   "outputs": [],
   "source": [
    "preprocessed[:500]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "NpHeMUMtpbuj"
   },
   "outputs": [],
   "source": [
    "params = (\"title\", \"editor\", \"words\", \"tokens\", \"unique\", \"dates\", \"irish\", \"langs\", \"author\", \"link\", \"raw_text\", \"text\", \"preprocessed\")\n",
    "json_string = entry_pattern % params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mzbDDuJ3pbuj"
   },
   "outputs": [],
   "source": [
    "json.loads(json_string)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "bJWD0SRJpbuj"
   },
   "outputs": [],
   "source": [
    "test = \"\"\"\n",
    "<h4>Canonical References</h4>\n",
    "<p>This text uses the <tt>DIV1</tt> element to represent the Year. <p>Refs: EVENT (<tt>&lt;DIV2&gt;</tt>)<p>This text uses the <tt>DIV1</tt> element to represent the Year. <h3>Profile Description</h3>\n",
    "<p>Created: By mostly unknown authors in Irish monastic scriptoria\n",
    "and medieval Irish clerics.\n",
    " Date range: c. 550-c.1588, various and mostly unknown.<h4>Use of language</h4>\n",
    "<h5>Language: [LA] A large portion of the earlier part of the annals is in Latin.</h5>\n",
    "<h5>Language: [GA] Most of the later annals are in Old, Middle, and Early Modern Irish with very many Latin words and formulae.</h5>\n",
    "<h5>Language: [EN] Witness-list in the front matter is in English.</h5>\n",
    "<h4>List of hands</h4>\n",
    "<p><b>H</b> [main] Ruaidhri &Oacute; Luin&iacute;n</p><p><b>H1</b> [interpolator/glossator] unknown</p><p><b>H2</b> [interpolator/glossator] Cathal Mac Maghnusa</p><p><b>H3</b> [interpolator/glossator] unknown</p><p><b>H4</b> [interpolator/glossator] unknown</p><p><b>LATE</b> [otherwise unidentified interpolator/glossator (vols 2-3)] unknown</p><h3>Revision History</h3>\n",
    "\"\"\"\n",
    "\n",
    "langs.findall(test)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "CELT_Crawler.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}