demo_fourth_crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def location_crawler(url1,r):\n",
    "    import re\n",
    "    import time\n",
    "    from urllib.parse import urljoin\n",
    "    from urllib.parse import urlparse\n",
    "    import requests\n",
    "    from bs4 import BeautifulSoup\n",
    "    valid_url = re.compile(r'https?:\\/\\/(www\\.)?[-a-zA-Z0-9@:%._\\+~#=]{2,256}\\.[a-z]{2,6}\\b([-a-zA-Z0-9@:%_\\+.~#?&//=]*)')\n",
    "    soup = BeautifulSoup(r.content,'lxml')\n",
    "    goto_location = soup.find('a',text=re.compile(r'Τοποθεσία|Που είμαστε|Πως θα μας βρείτε|Θα μας βρείτε|How to find us|Χάρτης|Χάρτες|πρόσβαση|Χάρτες & πρόσβαση|Χάρτης Google|^([Εε]πικοινωνία(?!.*επενδυτές).*$|[Cc]ontact [Us]?|Στοιχεία επικοινωνίας|Επικοινωνία|CONTACTS|Επικοινωνία|Επικοινωνείστε μαζί μας|INFO|ΕΠΙΚΟΙΝΩΝΙΑ|ΕΠΙΚΟΙΝΩΝΗΣΤΕ ΜΑΖΙ ΜΑΣ|ΕΠΙΚΟΙΝΩΝΙΑ|Επικοινωνια|[Cc]ontact|Contacts|CONTACT|[εΕ]πικοινωνια|ÎÏÎ¹ÎºÎ¿Î¹Î½ÏÎ½Î¯Î±|Επικοινωνία(?!.*επενδυτές).*$|Επικοινώνησε μαζί μας|^(ÎÏÎ¹ÎºÎ¿Î¹Î½ÏÎ½Î¹Î±)|ÎµÏÎ¹ÎºÎ¿Î¹Î½ÏÎ½Î¯Î±|ÅÐÉÊÏÉÍÙÍÉÁ|Πού θα μας βρείτε|[Cc]ontact|ÎÎ ÎÎÎÎÎÎ©ÎÎÎ|ΕΠΙΚ/ΝΙΑ|Επικοινωνήστε μαζί μας|Που είμαστε|ΕΠΙΚΟΙΝΩΝΙΑ|Επικοινωνια|Στοιχεία επικοινωνίας|Φόρμα Επικοινωνίας|Επικοινωνήστε μαζί μας|Επικοινωνήστε μαζί μας)'),attrs={'href' : re.compile(r'^(?!.*(javascript|mailto|Mailto))')})\n",
    "    if goto_location == None:\n",
    "        goto_location = soup.find(href = re.compile(r'[cC]ontact[-us]?\\b|CONTACT|Forma Epikoinwnias|communication|EPIKOINONIA|katastimata|know|contact_us|epik|ΕΠΙΚΟΙΝΩΝΙΑ|stoixeia-epikoinonias|Contacts|stores|communication|epikoinia|Contact|contact|[eE]pikoin[ow]nia\\b|contact|%C5%F0%E9%EA%EF%E9%ED%F9%ED%DF%E1|contac-us|epikoinwnia|where-we-are|find-us|2cont|contact|επικοινωνία|epikinonia|επικοινωνια|%ce%b5%cf%80%ce%b9%ce%ba%ce%bf%ce%b9%ce%bd%cf%89%ce%bd%ce%af%ce%b1|EPIKINONIA|contact-us|epikoinwnia|epikoinonia|Eπικοινωνία|Επικοινωνία|καταστήματα|επικοινωνια|epsilonpiiotakappaomicroniotanuomeganuiotaalpha|GrFeed|madres-oikodomikwn-ylikwn-volou.html|epikoinoneste-mazi-mas|/1/gr/8_3/8.htm'))\n",
    "    if goto_location !=None and 'mailto' not in goto_location['href'] and 'Mailto' not in goto_location['href'] and 'javascript' not in goto_location['href']:\n",
    "        if valid_url.search(goto_location['href']):\n",
    "            location_link = goto_location['href']\n",
    "        else:\n",
    "            location_link = urljoin(url1,goto_location['href'])\n",
    "    else:\n",
    "        location_link = url1\n",
    "    res = get_coordinates(location_link)\n",
    "    if res == []:\n",
    "        parsed = list(urlparse(url1))[1]\n",
    "        parsed = re.sub('www.','',parsed)\n",
    "        parsed = re.sub('.eu','',parsed)\n",
    "        parsed = re.sub('.com','',parsed)\n",
    "        parsed = re.sub('.gr','',parsed)\n",
    "        parsed = re.sub('.biz','',parsed)\n",
    "        parsed = re.sub('.net','',parsed)\n",
    "        parsed = re.sub('.me','',parsed)\n",
    "        parsed = re.sub('.webnode','',parsed)\n",
    "        res = find_place(parsed)\n",
    "        coords = res[1]\n",
    "        adrr = res[0]\n",
    "        tk = res[2]\n",
    "    else:\n",
    "        time.sleep(1.1)\n",
    "        adrr,tk = geolocate(res)\n",
    "        coords =','.join(res)\n",
    "    result = adrr+','+coords+','+tk\n",
    "    if result ==',,':\n",
    "        result = tk_search(location_link)\n",
    "    return result\n",
    "def get_coordinates(newlink):\n",
    "    from bs4 import BeautifulSoup\n",
    "    from selenium import webdriver\n",
    "    from selenium.common.exceptions import StaleElementReferenceException\n",
    "    import time \n",
    "    import re\n",
    "    import requests\n",
    "    path=\"C:\\\\Users\\\\User\\\\Desktop\\\\diploma thesis Data Science\\\\chromedriver.exe\"\n",
    "    driver=webdriver.Chrome(path)\n",
    "    #print(newlink)\n",
    "    driver.get(newlink)\n",
    "    time.sleep(5)\n",
    "    try:\n",
    "        cards = driver.find_elements_by_xpath(\"//div[@id]\")\n",
    "    except WebDriverException:\n",
    "        cards = []\n",
    "    except StaleElementReferenceException:\n",
    "        cards = []\n",
    "    #print(cards)\n",
    "    regexp = re.compile(r'http[s]?://maps.google.com/maps?')\n",
    "    hrefs = []\n",
    "    if cards!=[]:\n",
    "        for header in cards:\n",
    "            a = header.find_elements_by_xpath(\".//a[@target]\")\n",
    "            for cc in a:\n",
    "                #print(cc.get_attribute('href'))\n",
    "                if cc.get_attribute('href') ==None:\n",
    "                    continue\n",
    "                else:\n",
    "                    #print(cc.get_attribute('href'))\n",
    "                    if regexp.search(cc.get_attribute('href')):\n",
    "                        hrefs.append(cc.get_attribute('href'))\n",
    "    if hrefs == []:\n",
    "        r6 = requests.get(newlink)\n",
    "        soup6 = BeautifulSoup(r6.text,'lxml')\n",
    "        iframes = soup6.find_all('iframe',src=re.compile(r'http[s]?://maps.google.com/maps?'))\n",
    "        if iframes == []:\n",
    "            try:\n",
    "                iframes22 = soup6.find_all('iframe',src=re.compile(r'google'))\n",
    "                if iframes22 == []:\n",
    "                    hrefs = []\n",
    "                    lonlat = []\n",
    "                else:\n",
    "                    for frame in iframes22:\n",
    "                        tmp = re.findall(\"\\d{1,2}\\.\\d+\",frame['src'])\n",
    "                        if len(tmp) ==2:\n",
    "                            hrefs.append(tmp[0]+','+tmp[1])\n",
    "                        elif tmp == []:\n",
    "                            hrefs = []\n",
    "                        else:\n",
    "                            tmp67 = []\n",
    "                            for tm in tmp:\n",
    "                                if (float(tm)>=31 and float(tm)<=42):\n",
    "                                    tmp1 = tm\n",
    "                                if (float(tm)<=29 and float(tm)>=19.86):\n",
    "                                    tmp2 = tm\n",
    "                            #tmp = sorted(tmp67, key=float, reverse=True)\n",
    "                            #tm = [float(x) for x in tmp]\n",
    "                            #tmp1 = max(tm)\n",
    "                            #tm1 = [t for t in tm if t!=tmp1]\n",
    "                            #tmp2 = max(tm1)\n",
    "                            hrefs.append(str(tmp1)+','+str(tmp2))\n",
    "                    lonlat = hrefs\n",
    "            except ValueError:\n",
    "                divs = soup6.find('div',src=re.compile(r'google-maps-link'))\n",
    "                if divs !=None:\n",
    "                    tar = divs.find('a',href=re.compile(r'http[s]?://maps.google.com/maps?'))\n",
    "                    if tar !=None:\n",
    "                        hrefs = []\n",
    "                        hrefs.append(tar['href'])\n",
    "                        driver.quit()\n",
    "                        return clean_coords(hrefs)\n",
    "                    else:\n",
    "                        driver.quit()\n",
    "                        return []\n",
    "                else:\n",
    "                    driver.quit()\n",
    "                    return []\n",
    "        else:\n",
    "            for c in iframes:\n",
    "                hrefs.append(c['src'])\n",
    "            lonlat = clean_coords(hrefs)\n",
    "    else:\n",
    "        lonlat = clean_coords(hrefs)\n",
    "    driver.quit()\n",
    "    #lonlat = [re.sub('%2C',',',l) for l in lonlat]\n",
    "    lonlat = [','.join(re.findall(\"\\d+\\.\\d+\",l)) for l in lonlat]\n",
    "    lonlat = [re.sub('[^\\d\\.,]','',l).rstrip('.').lstrip('.').rstrip(',').lstrip(',') for l in lonlat]\n",
    "    if lonlat == [',,,'] or lonlat == [',']:\n",
    "        lonlat = []\n",
    "    return lonlat\n",
    "def clean_coords(hr):\n",
    "    returns = list(set(hr))\n",
    "    #res = [link[link.find('=')+1:link.find('&')].split(',') for link in returns]\n",
    "    res = []\n",
    "    for ss in returns:\n",
    "        try:\n",
    "            k = ss.split('ll=')[1]\n",
    "            res.append(k[:k.find('&')])\n",
    "        except IndexError:\n",
    "            k=ss.split('q=')[1]\n",
    "            res.append(k[:k.find('&')].replace('+',','))\n",
    "        except IndexError:\n",
    "            continue\n",
    "    return res\n",
    "def geolocate(coords):\n",
    "    from geopy.geocoders import Nominatim\n",
    "    import re\n",
    "    import time\n",
    "    import requests\n",
    "    apkey = 'your-key'\n",
    "    #geolocator = Nominatim(timeout=10)\n",
    "    add = []\n",
    "    pcode = []\n",
    "    for c in coords:\n",
    "        if c == '':\n",
    "            continue\n",
    "        else:\n",
    "            try:\n",
    "                c = re.sub('%2C',',',c)\n",
    "                k = c.split(',')\n",
    "                if float(k[1]) > float(k[0]):\n",
    "                    c = k[1]+ ','+ k[0]\n",
    "            except ValueError:\n",
    "                c = re.sub('%2C','',c)\n",
    "                k = c.split(',')\n",
    "                if float(k[1]) > float(k[0]):\n",
    "                    c = k[1]+ ','+ k[0]\n",
    "            except ValueError:\n",
    "                continue\n",
    "            url = 'https://maps.googleapis.com/maps/api/geocode/json?latlng='+c+'&key='+apkey\n",
    "            content = requests.get(url).json()\n",
    "            add.append(content['results'][0]['formatted_address'])\n",
    "            p1 = content['results'][0]['address_components']\n",
    "            pcode.append(p1[len(p1)-1]['short_name'])\n",
    "            #location = geolocator.reverse(c,timeout=10)\n",
    "            #if 'address' in location.raw.keys():\n",
    "            #    add.append(location.address)\n",
    "            #    add1 = location.raw['address']\n",
    "            #    if 'postcode' not in add1.keys():\n",
    "            #        pcode.append('')\n",
    "            #    else:\n",
    "            #        pcode.append(location.raw['address']['postcode'])\n",
    "            time.sleep(1.2)\n",
    "    retk = ''.join(pcode)\n",
    "    retad = ','.join(add)\n",
    "    return retad,retk\n",
    "def find_place(place_name):\n",
    "    from urllib.parse   import quote\n",
    "    import requests\n",
    "    import time\n",
    "    \n",
    "    api_key = 'api-key'\n",
    "    fname = place_name + ' '+ 'GREECE'\n",
    "    try:\n",
    "        url = 'https://maps.googleapis.com/maps/api/place/textsearch/json?query='+quote(fname)+'&key='+api_key\n",
    "        content = requests.get(url).json()\n",
    "    except:\n",
    "        time.sleep(5)\n",
    "        api2 = 'api-key'\n",
    "        url = 'https://maps.googleapis.com/maps/api/place/textsearch/json?query='+quote(fname)+'&key='+api2\n",
    "        content = requests.get(url).json()\n",
    "    if content['results'] != []:\n",
    "        address = content['results'][0]['formatted_address']\n",
    "        coordinates = str(content['results'][0]['geometry']['location']['lat'])+','+str(content['results'][0]['geometry']['location']['lng'])\n",
    "        tk = geocoding_forplaces(coordinates)\n",
    "        return [address,coordinates,tk]\n",
    "    else:\n",
    "        return ['','','']\n",
    "def geocoding_forplaces(coordinates):\n",
    "    import requests\n",
    "    geocoding_api1 = 'api-key'\n",
    "    url = 'https://maps.googleapis.com/maps/api/geocode/json?latlng='+coordinates+'&key='+geocoding_api1\n",
    "    content = requests.get(url).json()\n",
    "    p1 = content['results'][0]['address_components']\n",
    "    return p1[len(p1)-1]['short_name']\n",
    "import pandas as pd\n",
    "geo_data = pd.read_csv('zipcodes.txt', header = None)\n",
    "tks = list(geo_data[0])\n",
    "tks = [str(t) for t in tks]\n",
    "def tk_search(newlink):\n",
    "    import pandas as pd\n",
    "    import requests\n",
    "    import re\n",
    "    from bs4 import BeautifulSoup\n",
    "    from nltk.tokenize import RegexpTokenizer\n",
    "    from collections import Counter\n",
    "    r = requests.get(newlink)\n",
    "    soup = BeautifulSoup(r.content,'lxml')\n",
    "    texts = soup.findAll(text=True)\n",
    "    visible_texts = filter(visible, texts)\n",
    "    res = list(visible_texts)\n",
    "    totaltext = ' '.join(res)\n",
    "    tokenizer = RegexpTokenizer(r'\\w+')\n",
    "    tokens2 = tokenizer.tokenize(totaltext)\n",
    "    tokens3 = []\n",
    "    #print(tokens2)\n",
    "    found_tk = []\n",
    "    for tok in tokens2:\n",
    "        if tok in tks:\n",
    "            found_tk.append(tok)\n",
    "    if found_tk == []:\n",
    "        for first, second in zip(tokens2, tokens2[1:]):\n",
    "            tmpb = first+second\n",
    "            if tmpb in tks:\n",
    "                found_tk.append(tmpb)\n",
    "    finaltk = list(set(found_tk))\n",
    "    if finaltk !=[]:\n",
    "        add = []\n",
    "        for t in finaltk:\n",
    "            add.append(geocodetk(t))\n",
    "        return add[0]\n",
    "    else:\n",
    "        return ',,'\n",
    "def geocodetk(tks):\n",
    "    import requests\n",
    "    apkey = 'api-key'\n",
    "    url = 'https://maps.googleapis.com/maps/api/geocode/json?address='+tks+'GREECE&key='+apkey\n",
    "    r = requests.get(url).json()\n",
    "    location = r['results'][0]['geometry']['location']\n",
    "    coords = str(location['lat'])+','+str(location['lng'])\n",
    "    addr = r['results'][0][ 'formatted_address']\n",
    "    return addr+','+coords+','+tks\n",
    "def visible(element):\n",
    "    import re\n",
    "    if element.parent.name in ['style', 'script', '[document]', 'head', 'title']:\n",
    "        return False\n",
    "    elif re.match('<!--.*-->', str(element)):\n",
    "        return False\n",
    "    return True\n",
    "def check_redirects(url):\n",
    "    import requests\n",
    "    r = requests.get(url)\n",
    "    r1 = requests.get(r.url)\n",
    "    return r.url,r1\n",
    "def run_me(url):\n",
    "    url1,r = check_redirects(url)\n",
    "    print(location_crawler(url1,r))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Omirou, Egialia 251 00, Greece,38.258797,22.062338,251 00\n"
     ]
    }
   ],
   "source": [
    "run_me('http://couniniotis.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sapfous 29-35, Acharnes 136 71, Greece,38.0629975797081,23.73970891578256,136 71\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.veler.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EO53, Metaxades 680 10, Greece,41.4199,26.2241,680 10\n"
     ]
    }
   ],
   "source": [
    "run_me('http://hellasfrost.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Makedonomachon 9, Serres 621 25, Greece,41.081743,23.547921,621 25\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.agrimon.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Damianou 6, Acharnes 136 71, Greece,38.079156,23.746416,136 71\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.kallas-pap.com/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EO Volou Larisas, Volos 385 00, Greece,39.374696,22.897904,385 00\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.tsianakas.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EO Livadias Lamias, Livadia 321 00, Greece,38.474058770973386,22.91725906207575,321 00\n"
     ]
    }
   ],
   "source": [
    "run_me('http://rotosal.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3ο ΧΛΜ ΕΘΝΙΚΗΣ ΟΔΟΥ ΚΑΤΕΡΙΝΗΣ - ΛΑΡΙΣΑΣ, Katerini 601 00, Greece,40.24747499999999,22.533289,601 00\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.viozokat.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Leof. Andrea Siggrou 18, Athina 117 42, Greece,37.967979,23.7309127,117 42\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.biofresh-sa.com/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Eparch.Od. Neon Moudanion - Thessalonikis, Nea Propontida 630 80, Greece,40.387301,23.038009,630 80\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.domissima.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Iera Odos ke Naxou, Egaleo 122 42, Greece,37.98822759999999,23.6920481,122 42\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.pansil.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Alamanas 1, Marousi 151 25, Greece,38.043729,23.807108,151 25\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.hfo.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "EO Kavalas Xanthis, Nestos 642 00, Greece,40.97531902497841,24.59798222451531,642 00\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.tsarouchas.eu/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mezonos 22, Kalamata 241 00, Greece,37.032865,22.114345,241 00\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.andrianos.gr/gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Oraiokastro 570 13, Greece,40.7137773,22.9283587,57013\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.akrodria.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Athanasiou Diakou, Kimina 573 00, Greece,40.6156,22.6813,573 00\n"
     ]
    }
   ],
   "source": [
    "run_me('https://glaveris.gr/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Voriou Ipirou 78, Athina 104 44, Greece,37.997651,23.718259,104 44\n"
     ]
    }
   ],
   "source": [
    "run_me('http://www.spiroskatsaros.gr/')"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}