hh-rec-test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.729580198569\n",
      "0.957427107756\n"
     ]
    }
   ],
   "source": [
    "from scipy import spatial\n",
    "from sklearn.preprocessing import normalize\n",
    "\n",
    "a1 = [1, 1, 1, 1]\n",
    "a2 = [2, 54, 13, 15]\n",
    "a3 = [2, 2, 4, 3]\n",
    "print 1 - spatial.distance.cosine(a1, a2)\n",
    "print 1 - spatial.distance.cosine(a1, a3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200 OK\n",
      "200 OK\n",
      "{u'url': u'https://api.hh.ru/areas/1', u'id': u'1', u'name': u'\\u041c\\u043e\\u0441\\u043a\\u0432\\u0430'}\n",
      "[]\n",
      "[u'1.221', u'1.295', u'1.89']\n",
      "{u'currency': u'RUR', u'amount': 135000}\n",
      "{u'id': u'full', u'name': u'\\u041f\\u043e\\u043b\\u043d\\u0430\\u044f \\u0437\\u0430\\u043d\\u044f\\u0442\\u043e\\u0441\\u0442\\u044c'}\n",
      "[{u'id': u'fullDay', u'name': u'\\u041f\\u043e\\u043b\\u043d\\u044b\\u0439 \\u0434\\u0435\\u043d\\u044c'}, {u'id': u'remote', u'name': u'\\u0423\\u0434\\u0430\\u043b\\u0435\\u043d\\u043d\\u0430\\u044f \\u0440\\u0430\\u0431\\u043e\\u0442\\u0430'}]\n",
      "{u'months': 116}\n",
      "Java Developer\n",
      "Опытный разработчик программного обеспечения. Имею широкий спектр навыков, знаний и технологий. Разрабатываю быстрые программы, работающие с большими объемами данных в многопотоковой среде. Уделяю большое внимание качеству кода и удобству дальнейшей поддержки. Аналитически подхожу к решению задач. Имею хорошие знания алгоритмов и структур данных.\n",
      "\n",
      "Хобби, увлечения\n",
      "Интернет-технологии, путешествия, фотография, кулинария. \n",
      "Организовал и веду в местной школе кружок по программированию для детей.\n",
      "В последнее время интересуюсь машинным обучением и OpenCV\n",
      "\n",
      "Профиль на Habrahabr: http://habrahabr.ru/users/shurik2533/\n",
      "[u'Java', u'Java EE', u'Oracle Pl/SQL', u'Python', u'Data Analysis', u'Big Data', u'recommender systems', u'machine learning']\n",
      "[1, 2, 3]\n"
     ]
    }
   ],
   "source": [
    "import httplib\n",
    "import json\n",
    "headers = {\"Authorization\": \"Bearer PHUAM0L3PU56VNT041CJM3MUTGSABUCDTMBVHEAMF5CGCDEEIC7VFFT4VLOP0GQP\", \n",
    "           \"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"/resumes/mine\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "print r1.status, r1.reason\n",
    "data = r1.read()\n",
    "for i in range(len(json.loads(data)['items'])):\n",
    "    resume_id = json.loads(data)['items'][i]['id']\n",
    "    conn.request(\"GET\", \"/resumes/{0}\".format(resume_id), headers=headers)\n",
    "    r2 = conn.getresponse()\n",
    "    print r2.status, r2.reason\n",
    "    resume_data = r2.read()\n",
    "    features = [1,2,3]\n",
    "    print json.loads(resume_data)['area'] #по этому отсекаем сразу\n",
    "    print json.loads(resume_data)['relocation']['area'] #этот туда же\n",
    "    \n",
    "    \n",
    "    print [d['id'] for d in json.loads(resume_data)['specialization']] #для этого надо взять список всех специализаций. потом взять специализации по всем резюме и на их основе составить веса\n",
    "    print json.loads(resume_data)['salary']\n",
    "    print json.loads(resume_data)['employment']\n",
    "    print json.loads(resume_data)['schedules']\n",
    "    print json.loads(resume_data)['total_experience']\n",
    "    print json.loads(resume_data)['skill_set']\n",
    "    print json.loads(resume_data)['title'] #из этого достаем слова\n",
    "    print json.loads(resume_data)['skills']\n",
    "    \n",
    "    print features\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16823578\n",
      "found\n",
      "16823572\n",
      "found\n",
      "16823571\n",
      "found\n",
      "16823570\n",
      "found\n",
      "16823568\n",
      "found\n",
      "16823564\n",
      "found\n",
      "16823563\n",
      "found\n",
      "16823562\n",
      "found\n",
      "16823561\n",
      "found\n",
      "16823560\n",
      "found\n",
      "16823558\n",
      "found\n",
      "16823557\n",
      "found\n",
      "16823556\n",
      "found\n",
      "16823554\n",
      "found\n",
      "16823551\n",
      "found\n",
      "16823549\n",
      "found\n",
      "16823547\n",
      "found\n",
      "16823544\n",
      "found\n",
      "16823543\n",
      "found\n",
      "16823542\n",
      "found\n",
      "16823540\n",
      "found\n",
      "16823539\n",
      "found\n",
      "16823538\n",
      "found\n",
      "16823536\n",
      "found\n",
      "16823534\n",
      "found\n",
      "16823530\n",
      "found\n",
      "16823519\n",
      "found\n",
      "16823516\n",
      "found\n",
      "16823506\n",
      "found\n",
      "16823503\n",
      "found\n",
      "16823500\n",
      "found\n",
      "16823499\n",
      "found\n",
      "16823498\n",
      "found\n",
      "16823497\n",
      "found\n",
      "16823494\n",
      "found\n",
      "16823492\n",
      "found\n",
      "16823491\n",
      "found\n",
      "16823474\n",
      "found\n",
      "16823469\n",
      "found\n",
      "16823455\n",
      "found\n",
      "16823450\n",
      "found\n",
      "16823444\n",
      "found\n",
      "16823442\n",
      "found\n",
      "16823439\n",
      "found\n",
      "16823416\n",
      "found\n",
      "16823414\n",
      "found\n",
      "16823410\n",
      "found\n",
      "16823408\n",
      "found\n",
      "16823407\n",
      "found\n",
      "16823406\n",
      "found\n",
      "16823404\n",
      "found\n",
      "16823402\n",
      "found\n",
      "16823401\n",
      "found\n",
      "16823392\n",
      "found\n",
      "16823390\n",
      "found\n",
      "16823387\n",
      "found\n",
      "16823384\n",
      "found\n",
      "16823383\n",
      "found\n",
      "16823379\n",
      "found\n",
      "16823378\n",
      "found\n",
      "16823377\n",
      "found\n",
      "16823374\n",
      "found\n",
      "16823370\n",
      "found\n",
      "16823368\n",
      "found\n",
      "16823366\n",
      "found\n",
      "16823364\n",
      "found\n",
      "16823361\n",
      "found\n",
      "16823356\n",
      "found\n",
      "16823355\n",
      "found\n",
      "16823352\n",
      "found\n",
      "16823349\n",
      "found\n",
      "16823346\n",
      "found\n",
      "16823344\n",
      "found\n",
      "16823343\n",
      "found\n",
      "16823342\n",
      "found\n",
      "16823339\n",
      "found\n",
      "16823338\n",
      "found\n",
      "16823337\n",
      "found\n",
      "16823335\n",
      "found\n",
      "16823334\n",
      "found\n",
      "16823333\n",
      "found\n",
      "16823332\n",
      "found\n",
      "16823330\n",
      "found\n",
      "16823329\n",
      "found\n",
      "16823328\n",
      "found\n",
      "16823327\n",
      "found\n",
      "16823325\n",
      "found\n",
      "16823319\n",
      "found\n",
      "16823315\n",
      "found\n",
      "16823311\n",
      "found\n",
      "16823308\n",
      "found\n",
      "16823307\n",
      "found\n",
      "16823306\n",
      "found\n",
      "16823305\n",
      "found\n",
      "16823303\n",
      "found\n",
      "16823300\n",
      "found\n",
      "16823299\n",
      "found\n",
      "16823297\n",
      "found\n",
      "16823294\n",
      "found\n",
      "16823293\n",
      "found\n",
      "16823278\n",
      "found\n",
      "16823276\n",
      "found\n",
      "16823275\n",
      "found\n",
      "16823266\n",
      "found\n",
      "16823264\n",
      "found\n",
      "16823263\n",
      "found\n",
      "16823262\n",
      "found\n",
      "16823261\n",
      "found\n",
      "16823259\n",
      "found\n",
      "16823258\n",
      "found\n",
      "16823257\n",
      "found\n",
      "16823256\n",
      "found\n",
      "16823255\n",
      "found\n",
      "16823254\n",
      "found\n",
      "16823253\n",
      "found\n",
      "16823252\n",
      "found\n",
      "16823250\n",
      "found\n",
      "16823249\n",
      "found\n",
      "16823248\n",
      "found\n",
      "16823244\n",
      "found\n",
      "16823240\n",
      "found\n",
      "16823235\n",
      "found\n",
      "16823233\n",
      "found\n",
      "16823232\n",
      "found\n",
      "16823231\n",
      "found\n",
      "16823229\n",
      "found\n",
      "16823227\n",
      "found\n",
      "16823226\n",
      "found\n",
      "16823225\n",
      "found\n",
      "16823223\n",
      "found\n",
      "16823221\n",
      "found\n",
      "16823218\n",
      "found\n",
      "16823216\n",
      "found\n",
      "16823213\n",
      "found\n",
      "16823212\n",
      "found\n",
      "16823209\n",
      "found\n",
      "16823208\n",
      "found\n",
      "16823206\n",
      "found\n",
      "16823202\n",
      "found\n",
      "16823201\n",
      "found\n",
      "16823199\n",
      "found\n",
      "16823198\n",
      "found\n",
      "16823196\n",
      "found\n",
      "16823194\n",
      "found\n",
      "16823192\n",
      "found\n",
      "16823190\n",
      "found\n",
      "16823188\n",
      "found\n",
      "16823187\n",
      "found\n",
      "16823186\n",
      "found\n",
      "16823181\n",
      "found\n",
      "16823179\n",
      "found\n",
      "16823177\n",
      "found\n",
      "16823175\n",
      "found\n",
      "16823174\n",
      "found\n",
      "16823171\n",
      "found\n",
      "16823166\n",
      "found\n",
      "16823164\n",
      "found\n",
      "16823162\n",
      "found\n",
      "16823161\n",
      "found\n",
      "16823160\n",
      "found\n",
      "16823159\n",
      "found\n",
      "16823158\n",
      "found\n",
      "16823157\n",
      "found\n",
      "16823156\n",
      "found\n",
      "16823155\n",
      "found\n",
      "16823154\n",
      "found\n",
      "16823153\n",
      "found\n",
      "16823152\n",
      "found\n",
      "16823151\n",
      "found\n",
      "16823149\n",
      "found\n",
      "16823148\n",
      "found\n",
      "16823147\n",
      "found\n",
      "16823146\n",
      "found\n",
      "16823145\n",
      "found\n",
      "16823144\n",
      "found\n",
      "16823143\n",
      "found\n",
      "16823142\n",
      "found\n",
      "16823141\n",
      "found\n",
      "16823139\n",
      "found\n",
      "16823138\n",
      "found\n",
      "16823137\n",
      "found\n",
      "16823136\n",
      "found\n",
      "16823134\n",
      "found\n",
      "16823133\n",
      "found\n",
      "16823132\n",
      "found\n",
      "16823131\n",
      "found\n",
      "16823130\n",
      "found\n",
      "16823129\n",
      "found\n",
      "16823128\n",
      "found\n",
      "16823126\n",
      "found\n",
      "16823125\n",
      "found\n",
      "16823124\n",
      "found\n",
      "16823123\n",
      "found\n",
      "16823121\n",
      "found\n",
      "16823119\n",
      "found\n",
      "16823117\n",
      "found\n",
      "16823116\n",
      "found\n",
      "16823115\n",
      "found\n",
      "16823114\n",
      "found\n",
      "16823113\n",
      "found\n",
      "16823112\n",
      "found\n",
      "16823111\n",
      "found\n",
      "16823110\n",
      "found\n",
      "16823109\n",
      "found\n",
      "16823108\n",
      "found\n",
      "16823107\n",
      "found\n",
      "16823106\n",
      "found\n",
      "16823104\n",
      "found\n",
      "16823103\n",
      "found\n",
      "16823101\n",
      "found\n",
      "16823100\n",
      "found\n",
      "16823099\n",
      "found\n",
      "16823098\n",
      "found\n",
      "16823096\n",
      "found\n",
      "16823095\n",
      "found\n",
      "16823094\n",
      "found\n",
      "16823093\n",
      "found\n",
      "16823092\n",
      "found\n",
      "16823091\n",
      "found\n",
      "16823090\n",
      "found\n",
      "16823089\n",
      "found\n",
      "16823088\n",
      "found\n",
      "16823087\n",
      "found\n",
      "16823085\n",
      "found\n",
      "16823084\n",
      "found\n",
      "16823083\n",
      "found\n",
      "16823081\n",
      "found\n",
      "16823080\n",
      "found\n",
      "16823074\n",
      "found\n",
      "16823069\n",
      "found\n",
      "16823067\n",
      "found\n",
      "16823065\n",
      "found\n",
      "16823063\n",
      "found\n",
      "16823062\n",
      "found\n",
      "16823060\n",
      "found\n",
      "16823058\n",
      "found\n",
      "16823057\n",
      "found\n",
      "16823056\n",
      "found\n",
      "16823055\n",
      "found\n",
      "16823052\n",
      "found\n",
      "16823051\n",
      "found\n",
      "16823050\n",
      "found\n",
      "16823048\n",
      "found\n",
      "16823046\n",
      "found\n",
      "16823045\n",
      "found\n",
      "16823044\n",
      "found\n",
      "16823042\n",
      "found\n",
      "16823040\n",
      "found\n",
      "16823039\n",
      "found\n",
      "16823036\n",
      "found\n",
      "16823035\n",
      "found\n",
      "16823034\n",
      "found\n",
      "16823032\n",
      "found\n",
      "16823030\n",
      "found\n",
      "16823028\n",
      "found\n",
      "16823026\n",
      "found\n",
      "16823024\n",
      "found\n",
      "16823023\n",
      "found\n",
      "16823015\n",
      "found\n",
      "16823014\n",
      "found\n",
      "16823009\n",
      "found\n",
      "16823008\n",
      "found\n",
      "16823003\n",
      "found\n",
      "16823000\n"
     ]
    }
   ],
   "source": [
    "import httplib\n",
    "import json\n",
    "import pickle\n",
    "from tinydb import TinyDB\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "\n",
    "db = TinyDB('/home/shurik2533/vacancies_test.json')\n",
    "k = 0\n",
    "for i in range(16823582, 0, -1): #18822744\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    conn.request(\"GET\", \"/vacancies/{0}\".format(i), headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "    k = k+1\n",
    "    if r1.status==200:\n",
    "        vacancy_data = r1.read()\n",
    "        #print json.loads(vacancy_data)['archived']\n",
    "        if json.loads(vacancy_data)['archived'] == False:\n",
    "            db.insert(json.loads(vacancy_data))\n",
    "            print json.loads(vacancy_data)['id']\n",
    "            k=0\n",
    "    if i%1000 == 0:\n",
    "        print i\n",
    "        break;\n",
    "    if k == 100:\n",
    "        print \"break\"\n",
    "        break;\n",
    "    \n",
    "#print db.all()[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{u'7.14': u'\\u0410\\u0432\\u0442\\u043e\\u0437\\u0430\\u043f\\u0447\\u0430\\u0441\\u0442\\u0438', u'16.658': u'\\u041c\\u0443\\u043d\\u0438\\u0446\\u0438\\u043f\\u0430\\u043b\\u0438\\u0442\\u0435\\u0442', u'21.292': u'\\u0422\\u0430\\u043c\\u043e\\u0436\\u0435\\u043d\\u043d\\u043e\\u0435 \\u043e\\u0444\\u043e\\u0440\\u043c\\u043b\\u0435\\u043d\\u0438\\u0435', u'23.362': u'\\u042e\\u0440\\u0438\\u0441\\u043a\\u043e\\u043d\\u0441\\u0443\\u043b\\u044c\\u0442', u'6.336': u'\\u0423\\u0447\\u0435\\u0442 \\u043a\\u0430\\u0434\\u0440\\u043e\\u0432', u'23.165': u'\\u041d\\u0430\\u043b\\u043e\\u0433\\u043e\\u0432\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'5.61': u'\\u0414\\u0435\\u043d\\u0435\\u0436\\u043d\\u044b\\u0439 \\u0440\\u044b\\u043d\\u043e\\u043a (money market)', u'17.623': u'\\u0424\\u0438\\u043d\\u0430\\u043d\\u0441\\u043e\\u0432\\u044b\\u0435 \\u0443\\u0441\\u043b\\u0443\\u0433\\u0438', u'1.10': u'Web \\u043c\\u0430\\u0441\\u0442\\u0435\\u0440', u'3.166': u'\\u041d\\u0430\\u0440\\u0443\\u0436\\u043d\\u0430\\u044f \\u0440\\u0435\\u043a\\u043b\\u0430\\u043c\\u0430', u'5.367': u'\\u041c\\u0435\\u0442\\u043e\\u0434\\u043e\\u043b\\u043e\\u0433\\u0438\\u044f, \\u0411\\u0430\\u043d\\u043a\\u043e\\u0432\\u0441\\u043a\\u0438\\u0435 \\u0442\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438', u'5.366': u'\\u0420\\u0438\\u0441\\u043a\\u0438: \\u043f\\u0440\\u043e\\u0447\\u0438\\u0435', u'5.365': u'\\u041e\\u0446\\u0435\\u043d\\u043a\\u0430 \\u0437\\u0430\\u043b\\u043e\\u0433\\u0430, \\u0421\\u0442\\u043e\\u0438\\u043c\\u043e\\u0441\\u0442\\u0438 \\u0438\\u043c\\u0443\\u0449\\u0435\\u0441\\u0442\\u0432\\u0430', u'5.369': u'\\u041e\\u0442\\u0447\\u0435\\u0442\\u043d\\u043e\\u0441\\u0442\\u044c', u'5.368': u'\\u0420\\u0430\\u0437\\u0440\\u0430\\u0431\\u043e\\u0442\\u043a\\u0430 \\u043d\\u043e\\u0432\\u044b\\u0445 \\u043f\\u0440\\u043e\\u0434\\u0443\\u043a\\u0442\\u043e\\u0432, \\u041c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433', u'13.268': u'\\u0421\\u0435\\u0440\\u0442\\u0438\\u0444\\u0438\\u043a\\u0430\\u0446\\u0438\\u044f', u'22.518': u'\\u0421\\u043e\\u043c\\u0435\\u043b\\u044c\\u0435', u'22.104': u'\\u041a\\u0435\\u0439\\u0442\\u0435\\u0440\\u0438\\u043d\\u0433', u'29.560': u'\\u042d\\u043b\\u0435\\u043a\\u0442\\u0440\\u043e\\u043c\\u043e\\u043d\\u0442\\u0435\\u0440, \\u041a\\u0430\\u0431\\u0435\\u043b\\u044c\\u0449\\u0438\\u043a', u'29.561': u'\\u042e\\u0432\\u0435\\u043b\\u0438\\u0440', u'11.436': u'\\u041a\\u0438\\u043d\\u043e', u'18.57': u'\\u0413\\u043b\\u0430\\u0432\\u043d\\u044b\\u0439 \\u0438\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440', u'18.56': u'\\u0413\\u043b\\u0430\\u0432\\u043d\\u044b\\u0439 \\u0430\\u0433\\u0440\\u043e\\u043d\\u043e\\u043c', u'22.193': u'\\u041e\\u0444\\u0438\\u0446\\u0438\\u0430\\u043d\\u0442, \\u0411\\u0430\\u0440\\u043c\\u0435\\u043d', u'18.174': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'22.198': u'\\u041e\\u0440\\u0433\\u0430\\u043d\\u0438\\u0437\\u0430\\u0446\\u0438\\u044f \\u0432\\u0441\\u0442\\u0440\\u0435\\u0447, \\u041a\\u043e\\u043d\\u0444\\u0435\\u0440\\u0435\\u043d\\u0446\\u0438\\u0439', u'22.199': u'\\u041e\\u0440\\u0433\\u0430\\u043d\\u0438\\u0437\\u0430\\u0446\\u0438\\u044f \\u0442\\u0443\\u0440\\u0438\\u0441\\u0442\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u0445 \\u043f\\u0440\\u043e\\u0434\\u0443\\u043a\\u0442\\u043e\\u0432', u'7.455': u'\\u0428\\u0438\\u043d\\u044b, \\u0414\\u0438\\u0441\\u043a\\u0438', u'22.204': u'\\u041f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b \\u043a\\u0443\\u0445\\u043d\\u0438', u'15.308': u'\\u0422\\u0440\\u0430\\u043d\\u0441\\u043f\\u043e\\u0440\\u0442, \\u041b\\u043e\\u0433\\u0438\\u0441\\u0442\\u0438\\u043a\\u0430', u'14.60': u'\\u0413\\u0443\\u043c\\u0430\\u043d\\u0438\\u0442\\u0430\\u0440\\u043d\\u044b\\u0435 \\u043d\\u0430\\u0443\\u043a\\u0438', u'24.493': u'\\u041f\\u0430\\u0440\\u0438\\u043a\\u043c\\u0430\\u0445\\u0435\\u0440', u'24.492': u'\\u041c\\u0430\\u0441\\u0441\\u0430\\u0436\\u0438\\u0441\\u0442', u'9.22': u'\\u0410\\u0434\\u043c\\u0438\\u043d\\u0438\\u0441\\u0442\\u0440\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'19.421': u'\\u0410\\u043a\\u0442\\u0443\\u0430\\u0440\\u0438\\u0439', u'3.236': u'\\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e \\u0440\\u0435\\u043a\\u043b\\u0430\\u043c\\u044b', u'3.230': u'\\u041f\\u0440\\u043e\\u0434\\u0432\\u0438\\u0436\\u0435\\u043d\\u0438\\u0435, \\u0421\\u043f\\u0435\\u0446\\u0438\\u0430\\u043b\\u044c\\u043d\\u044b\\u0435 \\u043c\\u0435\\u0440\\u043e\\u043f\\u0440\\u0438\\u044f\\u0442\\u0438\\u044f', u'2.523': u'CIPA', u'12.326': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0440\\u043e\\u0435\\u043a\\u0442\\u0430\\u043c\\u0438', u'12.322': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0440\\u0430\\u043a\\u0442\\u0438\\u043a\\u043e\\u0439', u'16.435': u'\\u041d\\u0418\\u0418', u'11.62': u'\\u0414\\u0438\\u0437\\u0430\\u0439\\u043d, \\u0433\\u0440\\u0430\\u0444\\u0438\\u043a\\u0430, \\u0436\\u0438\\u0432\\u043e\\u043f\\u0438\\u0441\\u044c', u'17.269': u'\\u0422\\u0435\\u043b\\u0435\\u043a\\u043e\\u043c\\u043c\\u0443\\u043d\\u0438\\u043a\\u0430\\u0446\\u0438\\u0438, \\u0421\\u0435\\u0442\\u0435\\u0432\\u044b\\u0435 \\u0440\\u0435\\u0448\\u0435\\u043d\\u0438\\u044f', u'1.232': u'\\u041f\\u0440\\u043e\\u0434\\u044e\\u0441\\u0435\\u0440', u'5.121': u'\\u041a\\u043e\\u0440\\u043f\\u043e\\u0440\\u0430\\u0442\\u0438\\u0432\\u043d\\u043e\\u0435 \\u0444\\u0438\\u043d\\u0430\\u043d\\u0441\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'5.123': u'\\u041a\\u043e\\u0440\\u0440\\u0435\\u0441\\u043f\\u043e\\u043d\\u0434\\u0435\\u043d\\u0442\\u0441\\u043a\\u0438\\u0435, \\u041c\\u0435\\u0436\\u0434\\u0443\\u043d\\u0430\\u0440\\u043e\\u0434\\u043d\\u044b\\u0435 \\u043e\\u0442\\u043d\\u043e\\u0448\\u0435\\u043d\\u0438\\u044f', u'5.124': u'\\u0420\\u0438\\u0441\\u043a\\u0438: \\u043a\\u0440\\u0435\\u0434\\u0438\\u0442\\u043d\\u044b\\u0435', u'5.126': u'\\u041a\\u0440\\u0435\\u0434\\u0438\\u0442\\u044b', u'2.454': u'\\u041c\\u0421\\u0424\\u041e, IFRS', u'4.181': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'18.263': u'\\u0420\\u0443\\u043a\\u043e\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e \\u043f\\u0440\\u0435\\u0434\\u043f\\u0440\\u0438\\u044f\\u0442\\u0438\\u0435\\u043c', u'2.351': u'\\u0426\\u0435\\u043d\\u043d\\u044b\\u0435 \\u0431\\u0443\\u043c\\u0430\\u0433\\u0438', u'24.380': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0438', u'23.36': u'\\u0411\\u0430\\u043d\\u043a\\u043e\\u0432\\u0441\\u043a\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'3.305': u'\\u0422\\u043e\\u0440\\u0433\\u043e\\u0432\\u044b\\u0439 \\u043c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433(Trade marketing)', u'26.416': u'FMCG, \\u0422\\u043e\\u0432\\u0430\\u0440\\u044b \\u043d\\u0430\\u0440\\u043e\\u0434\\u043d\\u043e\\u0433\\u043e \\u043f\\u043e\\u0442\\u0440\\u0435\\u0431\\u043b\\u0435\\u043d\\u0438\\u044f', u'26.415': u'\\u0425\\u0438\\u043c\\u0438\\u0447\\u0435\\u0441\\u043a\\u0430\\u044f \\u043f\\u0440\\u043e\\u0434\\u0443\\u043a\\u0446\\u0438\\u044f', u'26.414': u'\\u042d\\u043b\\u0435\\u043a\\u0442\\u0440\\u043e\\u043d\\u0438\\u043a\\u0430, \\u0444\\u043e\\u0442\\u043e, \\u0432\\u0438\\u0434\\u0435\\u043e', u'26.413': u'\\u0422\\u043e\\u0432\\u0430\\u0440\\u044b \\u0434\\u043b\\u044f \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441\\u0430', u'26.412': u'\\u041f\\u0440\\u043e\\u0434\\u0443\\u043a\\u0442\\u044b \\u043f\\u0438\\u0442\\u0430\\u043d\\u0438\\u044f', u'26.411': u'\\u0424\\u0430\\u0440\\u043c\\u0430\\u0446\\u0435\\u0432\\u0442\\u0438\\u043a\\u0430', u'26.410': u'\\u041a\\u043e\\u043c\\u043f\\u044c\\u044e\\u0442\\u0435\\u0440\\u043d\\u0430\\u044f \\u0442\\u0435\\u0445\\u043d\\u0438\\u043a\\u0430', u'9.312': u'\\u0422\\u0443\\u0440\\u0438\\u0437\\u043c, \\u0413\\u043e\\u0441\\u0442\\u0438\\u043d\\u0438\\u0446\\u044b, \\u0420\\u0435\\u0441\\u0442\\u043e\\u0440\\u0430\\u043d\\u044b', u'9.317': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043c\\u0430\\u043b\\u044b\\u043c \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441\\u043e\\u043c', u'26.419': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'5.234': u'\\u041f\\u0440\\u043e\\u0435\\u043a\\u0442\\u043d\\u043e\\u0435 \\u0444\\u0438\\u043d\\u0430\\u043d\\u0441\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'1.420': u'\\u0410\\u0434\\u043c\\u0438\\u043d\\u0438\\u0441\\u0442\\u0440\\u0430\\u0442\\u043e\\u0440 \\u0431\\u0430\\u0437 \\u0434\\u0430\\u043d\\u043d\\u044b\\u0445', u'3.114': u'\\u041a\\u043e\\u043d\\u0441\\u0443\\u043b\\u044c\\u0442\\u0430\\u043d\\u0442', u'23.539': u'\\u0410\\u043d\\u0442\\u0438\\u043c\\u043e\\u043d\\u043e\\u043f\\u043e\\u043b\\u044c\\u043d\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'18.190': u'\\u041d\\u0435\\u0444\\u0442\\u0435\\u043f\\u0435\\u0440\\u0435\\u0440\\u0430\\u0431\\u043e\\u0442\\u043a\\u0430', u'14.37': u'\\u0411\\u0438\\u043e\\u0442\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438', u'15.389': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0438', u'15.388': u'\\u0410\\u0434\\u043c\\u0438\\u043d\\u0438\\u0441\\u0442\\u0440\\u0430\\u0442\\u0438\\u0432\\u043d\\u044b\\u0439 \\u043f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b', u'23.120': u'\\u041a\\u043e\\u0440\\u043f\\u043e\\u0440\\u0430\\u0442\\u0438\\u0432\\u043d\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'12.331': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0447\\u0435\\u0441\\u043a\\u043e\\u0435 \\u043a\\u043e\\u043d\\u0441\\u0443\\u043b\\u044c\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'11.134': u'\\u041b\\u0438\\u0442\\u0435\\u0440\\u0430\\u0442\\u0443\\u0440\\u043d\\u0430\\u044f, \\u0420\\u0435\\u0434\\u0430\\u043a\\u0442\\u043e\\u0440\\u0441\\u043a\\u0430\\u044f \\u0434\\u0435\\u044f\\u0442\\u0435\\u043b\\u044c\\u043d\\u043e\\u0441\\u0442\\u044c', u'5.241': u'\\u041f\\u0440\\u044f\\u043c\\u044b\\u0435 \\u0438\\u043d\\u0432\\u0435\\u0441\\u0442\\u0438\\u0446\\u0438\\u0438', u'5.534': u'\\u0424\\u0430\\u043a\\u0442\\u043e\\u0440\\u0438\\u043d\\u0433', u'18.299': u'\\u0422\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433, \\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e \\u0438 \\u043f\\u0435\\u0440\\u0435\\u0440\\u0430\\u0431\\u043e\\u0442\\u043a\\u0430 \\u0437\\u0435\\u0440\\u043d\\u043e\\u0432\\u044b\\u0445', u'18.290': u'\\u0421\\u0442\\u0440\\u043e\\u0439\\u043c\\u0430\\u0442\\u0435\\u0440\\u0438\\u0430\\u043b\\u044b', u'14.178': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'18.297': u'\\u0422\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433', u'18.13': u'\\u0410\\u0432\\u0438\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'13.438': u'\\u041e\\u043f\\u0442\\u0438\\u043a\\u0430', u'18.16': u'\\u0410\\u0432\\u0442\\u043e\\u043c\\u043e\\u0431\\u0438\\u043b\\u044c\\u043d\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'13.433': u'\\u0420\\u0435\\u0433\\u0438\\u0441\\u0442\\u0440\\u0430\\u0442\\u0443\\u0440\\u0430', u'13.432': u'\\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e', u'6.247': u'\\u0420\\u0430\\u0437\\u0432\\u0438\\u0442\\u0438\\u0435 \\u043f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b\\u0430', u'17.441': u'\\u0424\\u0440\\u0430\\u043d\\u0447\\u0430\\u0439\\u0437\\u0438\\u043d\\u0433', u'17.443': u'\\u041c\\u0435\\u0431\\u0435\\u043b\\u044c', u'17.446': u'\\u0421\\u0438\\u0441\\u0442\\u0435\\u043c\\u044b \\u0431\\u0435\\u0437\\u043e\\u043f\\u0430\\u0441\\u043d\\u043e\\u0441\\u0442\\u0438', u'1.161': u'\\u041c\\u0443\\u043b\\u044c\\u0442\\u0438\\u043c\\u0435\\u0434\\u0438\\u0430', u'18.129': u'\\u041b\\u0435\\u0433\\u043a\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'21.310': u'\\u0422\\u0440\\u0443\\u0431\\u043e\\u043f\\u0440\\u043e\\u0432\\u043e\\u0434\\u044b', u'22.329': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u0440\\u0435\\u0441\\u0442\\u043e\\u0440\\u0430\\u043d\\u0430\\u043c\\u0438, \\u0411\\u0430\\u0440\\u0430\\u043c\\u0438', u'15.93': u'\\u0418\\u043d\\u0444\\u043e\\u0440\\u043c\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u044b\\u0435 \\u0442\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438, \\u0418\\u043d\\u0442\\u0435\\u0440\\u043d\\u0435\\u0442, \\u041c\\u0443\\u043b\\u044c\\u0442\\u0438\\u043c\\u0435\\u0434\\u0438\\u0430', u'22.175': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'2.179': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'23.88': u'\\u0418\\u043d\\u0442\\u0435\\u043b\\u043b\\u0435\\u043a\\u0442\\u0443\\u0430\\u043b\\u044c\\u043d\\u0430\\u044f \\u0441\\u043e\\u0431\\u0441\\u0442\\u0432\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'20.396': u'\\u042d\\u043a\\u0441\\u043f\\u043b\\u0443\\u0430\\u0442\\u0430\\u0446\\u0438\\u044f', u'13.489': u'\\u041c\\u0435\\u0434\\u0438\\u0446\\u0438\\u043d\\u0441\\u043a\\u0438\\u0439 \\u043f\\u0440\\u0435\\u0434\\u0441\\u0442\\u0430\\u0432\\u0438\\u0442\\u0435\\u043b\\u044c', u'6.107': u'\\u041a\\u043e\\u043c\\u043f\\u0435\\u043d\\u0441\\u0430\\u0446\\u0438\\u0438 \\u0438 \\u043b\\u044c\\u0433\\u043e\\u0442\\u044b', u'9.532': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u0437\\u0430\\u043a\\u0443\\u043f\\u043a\\u0430\\u043c\\u0438', u'5.4': u'Forex', u'18.81': u'\\u0418\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440', u'18.86': u'\\u0418\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440, \\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e \\u0441\\u0430\\u0445\\u0430\\u0440\\u0430', u'7.566': u'\\u0410\\u0432\\u0442\\u043e\\u043c\\u043e\\u0439\\u0449\\u0438\\u043a', u'7.565': u'\\u0410\\u0432\\u0442\\u043e\\u0436\\u0435\\u0441\\u0442\\u044f\\u043d\\u0449\\u0438\\u043a', u'18.85': u'\\u0418\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440, \\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e \\u0438 \\u043f\\u0435\\u0440\\u0435\\u0440\\u0430\\u0431\\u043e\\u0442\\u043a\\u0430 \\u0437\\u0435\\u0440\\u043d\\u043e\\u0432\\u044b\\u0445', u'8.575': u'\\u041f\\u043e\\u0436\\u0430\\u0440\\u043d\\u0430\\u044f \\u0431\\u0435\\u0437\\u043e\\u043f\\u0430\\u0441\\u043d\\u043e\\u0441\\u0442\\u044c', u'27.533': u'\\u041f\\u043e\\u043c\\u043e\\u0449\\u043d\\u0438\\u043a \\u043f\\u043e \\u0445\\u043e\\u0437\\u044f\\u0439\\u0441\\u0442\\u0432\\u0443, \\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u044f\\u044e\\u0449\\u0438\\u0439', u'10.472': u'\\u0413\\u0430\\u0437', u'3.40': u'\\u0411\\u0440\\u0435\\u043d\\u0434-\\u043c\\u0435\\u043d\\u0435\\u0434\\u0436\\u043c\\u0435\\u043d\\u0442', u'26.427': u'\\u041c\\u0435\\u0442\\u0430\\u043b\\u043b\\u043e\\u043f\\u0440\\u043e\\u043a\\u0430\\u0442', u'5.399': u'\\u0418\\u043f\\u043e\\u0442\\u0435\\u043a\\u0430, \\u0418\\u043f\\u043e\\u0442\\u0435\\u0447\\u043d\\u043e\\u0435 \\u043a\\u0440\\u0435\\u0434\\u0438\\u0442\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'3.48': u'\\u0412\\u0435\\u0440\\u0441\\u0442\\u0430\\u043b\\u044c\\u0449\\u0438\\u043a', u'12.376': u'\\u041d\\u0435\\u0434\\u0432\\u0438\\u0436\\u0438\\u043c\\u043e\\u0441\\u0442\\u044c', u'18.339': u'\\u0424\\u0430\\u0440\\u043c\\u0430\\u0446\\u0435\\u0432\\u0442\\u0438\\u0447\\u0435\\u0441\\u043a\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'15.167': u'\\u041d\\u0430\\u0443\\u043a\\u0430, \\u041e\\u0431\\u0440\\u0430\\u0437\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'13.155': u'\\u041c\\u043b\\u0430\\u0434\\u0448\\u0438\\u0439 \\u0438 \\u0441\\u0440\\u0435\\u0434\\u043d\\u0438\\u0439 \\u043c\\u0435\\u0434\\u043f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b', u'1.89': u'\\u0418\\u043d\\u0442\\u0435\\u0440\\u043d\\u0435\\u0442', u'17.59': u'\\u0413\\u0421\\u041c, \\u043d\\u0435\\u0444\\u0442\\u044c, \\u0431\\u0435\\u043d\\u0437\\u0438\\u043d', u'12.97': u'\\u0418\\u0441\\u0441\\u043b\\u0435\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u044f \\u0440\\u044b\\u043d\\u043a\\u0430', u'5.192': u'\\u041e\\u0431\\u043c\\u0435\\u043d\\u043d\\u044b\\u0435 \\u043f\\u0443\\u043d\\u043a\\u0442\\u044b, \\u0411\\u0430\\u043d\\u043a\\u043e\\u043c\\u0430\\u0442\\u044b', u'23.72': u'\\u0417\\u0430\\u043a\\u043e\\u043d\\u043e\\u0442\\u0432\\u043e\\u0440\\u0447\\u0435\\u0441\\u0442\\u0432\\u043e', u'5.195': u'\\u041e\\u041f\\u0415\\u0420\\u0423', u'19.18': u'\\u0410\\u0432\\u0442\\u043e\\u0441\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'5.691': u'Private Banking', u'17.538': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0438 \\u043f\\u043e \\u0442\\u0435\\u043b\\u0435\\u0444\\u043e\\u043d\\u0443, \\u0422\\u0435\\u043b\\u0435\\u043c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433', u'5.177': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'23.476': u'\\u0417\\u0435\\u043c\\u0435\\u043b\\u044c\\u043d\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'23.477': u'\\u0414\\u043e\\u0433\\u043e\\u0432\\u043e\\u0440\\u043d\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'23.478': u'\\u0420\\u0435\\u0433\\u0438\\u0441\\u0442\\u0440\\u0430\\u0446\\u0438\\u044f \\u044e\\u0440\\u0438\\u0434\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u0445 \\u043b\\u0438\\u0446', u'23.479': u'\\u0412\\u0437\\u044b\\u0441\\u043a\\u0430\\u043d\\u0438\\u0435 \\u0437\\u0430\\u0434\\u043e\\u043b\\u0436\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u0438, \\u041a\\u043e\\u043b\\u043b\\u0435\\u043a\\u0442\\u043e\\u0440\\u0441\\u043a\\u0430\\u044f \\u0434\\u0435\\u044f\\u0442\\u0435\\u043b\\u044c\\u043d\\u043e\\u0441\\u0442\\u044c', u'14.141': u'\\u041c\\u0430\\u0442\\u0435\\u043c\\u0430\\u0442\\u0438\\u043a\\u0430', u'17.535': u'\\u0422\\u043e\\u0440\\u0433\\u043e\\u0432\\u044b\\u0435 \\u0441\\u0435\\u0442\\u0438', u'11.173': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'24.377': u'\\u041a\\u043e\\u0441\\u043c\\u0435\\u0442\\u043e\\u043b\\u043e\\u0433\\u0438\\u044f', u'12.7': u'PR Consulting', u'9.105': u'\\u041a\\u043e\\u043c\\u043c\\u0435\\u0440\\u0447\\u0435\\u0441\\u043a\\u0438\\u0439 \\u0411\\u0430\\u043d\\u043a', u'12.5': u'Internet, E-Commerce', u'4.32': u'\\u0410\\u0440\\u0445\\u0438\\u0432\\u0438\\u0441\\u0442', u'24.378': u'\\u0422\\u0440\\u0435\\u043d\\u0435\\u0440\\u0441\\u043a\\u0438\\u0439 \\u0441\\u043e\\u0441\\u0442\\u0430\\u0432', u'24.379': u'\\u0410\\u0434\\u043c\\u0438\\u043d\\u0438\\u0441\\u0442\\u0440\\u0430\\u0446\\u0438\\u044f', u'20.58': u'\\u0413\\u043e\\u0441\\u0442\\u0438\\u043d\\u0438\\u0446\\u044b, \\u041c\\u0430\\u0433\\u0430\\u0437\\u0438\\u043d\\u044b', u'21.506': u'\\u042d\\u043a\\u0441\\u043f\\u0435\\u0434\\u0438\\u0442\\u043e\\u0440', u'21.482': u'\\u0412\\u043e\\u0434\\u0438\\u0442\\u0435\\u043b\\u044c', u'22.55': u'\\u0413\\u0438\\u0434, \\u042d\\u043a\\u0441\\u043a\\u0443\\u0440\\u0441\\u043e\\u0432\\u043e\\u0434', u'21.481': u'\\u0414\\u0438\\u0441\\u043f\\u0435\\u0442\\u0447\\u0435\\u0440', u'19.284': u'\\u0421\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u043d\\u0435\\u0434\\u0432\\u0438\\u0436\\u0438\\u043c\\u043e\\u0441\\u0442\\u0438', u'9.448': u'\\u042e\\u0440\\u0438\\u0441\\u043f\\u0440\\u0443\\u0434\\u0435\\u043d\\u0446\\u0438\\u044f', u'5.579': u'\\u0420\\u0430\\u0431\\u043e\\u0442\\u0430 \\u0441 \\u043f\\u0440\\u043e\\u0431\\u043b\\u0435\\u043c\\u043d\\u044b\\u043c\\u0438 \\u0437\\u0430\\u0435\\u043c\\u0449\\u0438\\u043a\\u0430\\u043c\\u0438', u'20.233': u'\\u041f\\u0440\\u043e\\u0435\\u043a\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435, \\u0410\\u0440\\u0445\\u0438\\u0442\\u0435\\u043a\\u0442\\u0443\\u0440\\u0430', u'5.133': u'\\u0420\\u0438\\u0441\\u043a\\u0438: \\u043b\\u0438\\u0437\\u0438\\u043d\\u0433\\u043e\\u0432\\u044b\\u0435', u'10.315': u'\\u0423\\u0433\\u043e\\u043b\\u044c', u'10.258': u'\\u0420\\u0443\\u0434\\u0430', u'4.264': u'\\u0421\\u0435\\u043a\\u0440\\u0435\\u0442\\u0430\\u0440\\u044c', u'13.578': u'\\u0424\\u0430\\u0440\\u043c\\u0430\\u0446\\u0435\\u0432\\u0442', u'5.304': u'\\u0422\\u043e\\u0440\\u0433\\u043e\\u0432\\u043e\\u0435 \\u0444\\u0438\\u043d\\u0430\\u043d\\u0441\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'9.67': u'\\u0414\\u043e\\u0431\\u044b\\u0447\\u0430 c\\u044b\\u0440\\u044c\\u044f', u'17.401': u'\\u0425\\u0438\\u043c\\u0438\\u0447\\u0435\\u0441\\u043a\\u0430\\u044f \\u043f\\u0440\\u043e\\u0434\\u0443\\u043a\\u0446\\u0438\\u044f', u'5.51': u'\\u0412\\u043d\\u0443\\u0442\\u0440\\u0435\\u043d\\u043d\\u0438\\u0435 \\u043e\\u043f\\u0435\\u0440\\u0430\\u0446\\u0438\\u0438 (Back Office)', u'22.353': u'\\u0428\\u0435\\u0444-\\u043f\\u043e\\u0432\\u0430\\u0440', u'3.171': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c/\\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'12.180': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'23.276': u'\\u0421\\u043b\\u0438\\u044f\\u043d\\u0438\\u044f \\u0438 \\u043f\\u043e\\u0433\\u043b\\u043e\\u0449\\u0435\\u043d\\u0438\\u044f', u'22.491': u'\\u041f\\u043e\\u0432\\u0430\\u0440', u'18.212': u'\\u041f\\u043e\\u043b\\u0438\\u0433\\u0440\\u0430\\u0444\\u0438\\u044f', u'19.259': u'\\u0420\\u0443\\u043a\\u043e\\u0432\\u043e\\u0434\\u0438\\u0442\\u0435\\u043b\\u044c \\u043d\\u0430\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u044f', u'21.564': u'\\u0420\\u0430\\u0431\\u043e\\u0447\\u0438\\u0439 \\u0441\\u043a\\u043b\\u0430\\u0434\\u0430', u'1.395': u'\\u0411\\u0430\\u043d\\u043a\\u043e\\u0432\\u0441\\u043a\\u043e\\u0435 \\u041f\\u041e', u'5.370': u'\\u041a\\u0430\\u0437\\u043d\\u0430\\u0447\\u0435\\u0439\\u0441\\u0442\\u0432\\u043e, \\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043b\\u0438\\u043a\\u0432\\u0438\\u0434\\u043d\\u043e\\u0441\\u0442\\u044c\\u044e', u'23.286': u'\\u0421\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'5.372': u'\\u0411\\u044e\\u0434\\u0436\\u0435\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'29.495': u'\\u0413\\u0440\\u0443\\u0437\\u0447\\u0438\\u043a', u'16.216': u'\\u041f\\u0440\\u0430\\u0432\\u0438\\u0442\\u0435\\u043b\\u044c\\u0441\\u0442\\u0432\\u043e', u'22.505': u'\\u0428\\u0432\\u0435\\u0439\\u0446\\u0430\\u0440', u'21.136': u'\\u041b\\u043e\\u0433\\u0438\\u0441\\u0442\\u0438\\u043a\\u0430', u'1.137': u'\\u041c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433', u'9.145': u'\\u041c\\u0435\\u0434\\u0438\\u0446\\u0438\\u043d\\u0430, \\u0424\\u0430\\u0440\\u043c\\u0430\\u0446\\u0435\\u0432\\u0442\\u0438\\u043a\\u0430', u'8.77': u'\\u0418\\u043c\\u0443\\u0449\\u0435\\u0441\\u0442\\u0432\\u0435\\u043d\\u043d\\u0430\\u044f \\u0431\\u0435\\u0437\\u043e\\u043f\\u0430\\u0441\\u043d\\u043e\\u0441\\u0442\\u044c', u'29.513': u'\\u0421\\u0442\\u043e\\u043b\\u044f\\u0440', u'29.512': u'\\u0424\\u0430\\u0441\\u043e\\u0432\\u0449\\u0438\\u043a', u'2.33': u'\\u0410\\u0443\\u0434\\u0438\\u0442', u'29.515': u'\\u042d\\u043b\\u0435\\u043a\\u0442\\u0440\\u0438\\u043a', u'8.202': u'\\u041e\\u0445\\u0440\\u0430\\u043d\\u043d\\u0438\\u043a', u'16.38': u'\\u0411\\u043b\\u0430\\u0433\\u043e\\u0442\\u0432\\u043e\\u0440\\u0438\\u0442\\u0435\\u043b\\u044c\\u043d\\u043e\\u0441\\u0442\\u044c', u'13.587': u'\\u0412\\u0440\\u0430\\u0447-\\u044d\\u043a\\u0441\\u043f\\u0435\\u0440\\u0442', u'29.516': u'\\u0428\\u0432\\u0435\\u044f', u'15.363': u'\\u042e\\u0440\\u0438\\u0441\\u0442\\u044b', u'21.74': u'\\u0417\\u0430\\u043a\\u0443\\u043f\\u043a\\u0438, \\u0421\\u043d\\u0430\\u0431\\u0436\\u0435\\u043d\\u0438\\u0435', u'19.430': u'\\u0423\\u0440\\u0435\\u0433\\u0443\\u043b\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u0443\\u0431\\u044b\\u0442\\u043a\\u043e\\u0432', u'24.624': u'\\u041d\\u043e\\u0433\\u0442\\u0435\\u0432\\u043e\\u0439 \\u0441\\u0435\\u0440\\u0432\\u0438\\u0441', u'2.125': u'\\u041a\\u0440\\u0435\\u0434\\u0438\\u0442\\u043d\\u044b\\u0439 \\u043a\\u043e\\u043d\\u0442\\u0440\\u043e\\u043b\\u044c', u'3.90': u'\\u0418\\u043d\\u0442\\u0435\\u0440\\u043d\\u0435\\u0442-\\u043c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433', u'7.239': u'\\u041f\\u0440\\u043e\\u043a\\u0430\\u0442, \\u043b\\u0438\\u0437\\u0438\\u043d\\u0433', u'9.321': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b\\u043e\\u043c, \\u0422\\u0440\\u0435\\u043d\\u0438\\u043d\\u0433\\u0438', u'3.98': u'\\u0418\\u0441\\u0441\\u043b\\u0435\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u044f \\u0440\\u044b\\u043d\\u043a\\u0430', u'14.349': u'\\u0425\\u0438\\u043c\\u0438\\u044f', u'7.235': u'\\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e', u'11.71': u'\\u0416\\u0443\\u0440\\u043d\\u0430\\u043b\\u0438\\u0441\\u0442\\u0438\\u043a\\u0430', u'11.76': u'\\u0418\\u0437\\u0434\\u0430\\u0442\\u0435\\u043b\\u044c\\u0441\\u043a\\u0430\\u044f \\u0434\\u0435\\u044f\\u0442\\u0435\\u043b\\u044c\\u043d\\u043e\\u0441\\u0442\\u044c', u'19.19': u'\\u0410\\u0433\\u0435\\u043d\\u0442', u'14.340': u'\\u0424\\u0438\\u0437\\u0438\\u043a\\u0430', u'12.197': u'\\u041e\\u0440\\u0433\\u0430\\u043d\\u0438\\u0437\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u043e\\u0435 \\u043a\\u043e\\u043d\\u0441\\u0443\\u043b\\u044c\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'1.221': u'\\u041f\\u0440\\u043e\\u0433\\u0440\\u0430\\u043c\\u043c\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435, \\u0420\\u0430\\u0437\\u0440\\u0430\\u0431\\u043e\\u0442\\u043a\\u0430', u'1.225': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0438', u'5.132': u'\\u041b\\u0438\\u0437\\u0438\\u043d\\u0433', u'20.573': u'\\u0422\\u0435\\u043d\\u0434\\u0435\\u0440\\u044b', u'15.730': u'\\u0411\\u0443\\u0445\\u0433\\u0430\\u043b\\u0442\\u0435\\u0440\\u0438\\u044f', u'19.282': u'\\u0421\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441\\u0430', u'19.283': u'\\u0421\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u0436\\u0438\\u0437\\u043d\\u0438', u'2.463': u'\\u0411\\u0443\\u0445\\u0433\\u0430\\u043b\\u0442\\u0435\\u0440-\\u043a\\u0430\\u043b\\u044c\\u043a\\u0443\\u043b\\u044f\\u0442\\u043e\\u0440', u'22.11': u'\\u0410\\u0432\\u0438\\u0430\\u0431\\u0438\\u043b\\u0435\\u0442\\u044b', u'2.467': u'ACCA', u'2.465': u'GAAP', u'2.464': u'\\u041e\\u0441\\u043d\\u043e\\u0432\\u043d\\u044b\\u0435 \\u0441\\u0440\\u0435\\u0434\\u0441\\u0442\\u0432\\u0430', u'2.342': u'\\u0424\\u0438\\u043d\\u0430\\u043d\\u0441\\u043e\\u0432\\u044b\\u0439 \\u0430\\u043d\\u0430\\u043b\\u0438\\u0437', u'2.343': u'\\u0424\\u0438\\u043d\\u0430\\u043d\\u0441\\u043e\\u0432\\u044b\\u0439 \\u043a\\u043e\\u043d\\u0442\\u0440\\u043e\\u043b\\u044c', u'2.469': u'\\u041f\\u0435\\u0440\\u0432\\u0438\\u0447\\u043d\\u0430\\u044f \\u0434\\u043e\\u043a\\u0443\\u043c\\u0435\\u043d\\u0442\\u0430\\u0446\\u0438\\u044f', u'2.468': u'\\u0422\\u041c\\u0426', u'2.344': u'\\u0424\\u0438\\u043d\\u0430\\u043d\\u0441\\u043e\\u0432\\u044b\\u0439 \\u043c\\u0435\\u043d\\u0435\\u0434\\u0436\\u043c\\u0435\\u043d\\u0442', u'11.99': u'\\u041a\\u0430\\u0437\\u0438\\u043d\\u043e \\u0438 \\u0438\\u0433\\u043e\\u0440\\u043d\\u044b\\u0439 \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441', u'3.318': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433\\u043e\\u043c', u'3.253': u'\\u0420\\u0435\\u043a\\u043b\\u0430\\u043c\\u043d\\u044b\\u0439 \\u0430\\u0433\\u0435\\u043d\\u0442', u'23.21': u'\\u0410\\u0434\\u0432\\u043e\\u043a\\u0430\\u0442', u'23.29': u'\\u0410\\u0440\\u0431\\u0438\\u0442\\u0440\\u0430\\u0436', u'20.83': u'\\u0418\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440', u'20.375': u'\\u0414\\u0435\\u0432\\u0435\\u043b\\u043e\\u043f\\u0435\\u0440', u'17.303': u'\\u0422\\u043e\\u0440\\u0433\\u043e\\u0432\\u043b\\u044f \\u0431\\u0438\\u0440\\u0436\\u0435\\u0432\\u044b\\u043c\\u0438 \\u0442\\u043e\\u0432\\u0430\\u0440\\u0430\\u043c\\u0438', u'17.302': u'FMCG, \\u0422\\u043e\\u0432\\u0430\\u0440\\u044b \\u043d\\u0430\\u0440\\u043e\\u0434\\u043d\\u043e\\u0433\\u043e \\u043f\\u043e\\u0442\\u0440\\u0435\\u0431\\u043b\\u0435\\u043d\\u0438\\u044f', u'17.301': u'\\u0422\\u043e\\u0432\\u0430\\u0440\\u044b \\u0434\\u043b\\u044f \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441\\u0430', u'17.306': u'\\u0422\\u043e\\u0440\\u0433\\u043e\\u0432\\u044b\\u0439 \\u043f\\u0440\\u0435\\u0434\\u0441\\u0442\\u0430\\u0432\\u0438\\u0442\\u0435\\u043b\\u044c', u'5.224': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0430 \\u0444\\u0438\\u043d\\u0430\\u043d\\u0441\\u043e\\u0432\\u044b\\u0445 \\u043f\\u0440\\u043e\\u0434\\u0443\\u043a\\u0442\\u043e\\u0432', u'5.341': u'\\u0424\\u0438\\u043b\\u0438\\u0430\\u043b\\u044b', u'14.87': u'\\u0418\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440\\u043d\\u044b\\u0435 \\u043d\\u0430\\u0443\\u043a\\u0438', u'17.111': u'\\u041a\\u043e\\u043c\\u043f\\u044c\\u044e\\u0442\\u0435\\u0440\\u043d\\u0430\\u044f \\u0442\\u0435\\u0445\\u043d\\u0438\\u043a\\u0430', u'1.296': u'\\u0422\\u0435\\u0445\\u043d\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u0439 \\u043f\\u0438\\u0441\\u0430\\u0442\\u0435\\u043b\\u044c', u'1.295': u'\\u0422\\u0435\\u043b\\u0435\\u043a\\u043e\\u043c\\u043c\\u0443\\u043d\\u0438\\u043a\\u0430\\u0446\\u0438\\u0438', u'4.47': u'\\u0412\\u0432\\u043e\\u0434 \\u0434\\u0430\\u043d\\u043d\\u044b\\u0445', u'1.359': u'\\u042d\\u043b\\u0435\\u043a\\u0442\\u0440\\u043e\\u043d\\u043d\\u0430\\u044f \\u043a\\u043e\\u043c\\u043c\\u0435\\u0440\\u0446\\u0438\\u044f', u'15.146': u'\\u041c\\u0435\\u0434\\u0438\\u0446\\u0438\\u043d\\u0430, \\u0424\\u0430\\u0440\\u043c\\u0430\\u0446\\u0435\\u0432\\u0442\\u0438\\u043a\\u0430', u'4.127': u'\\u041a\\u0443\\u0440\\u044c\\u0435\\u0440', u'18.451': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0440\\u043e\\u0435\\u043a\\u0442\\u0430\\u043c\\u0438', u'18.453': u'\\u042e\\u0432\\u0435\\u043b\\u0438\\u0440\\u043d\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'10.323': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0440\\u0435\\u0434\\u043f\\u0440\\u0438\\u044f\\u0442\\u0438\\u0435\\u043c', u'17.572': u'\\u0422\\u0435\\u043d\\u0434\\u0435\\u0440\\u044b', u'1.3': u'CTO, CIO, \\u0414\\u0438\\u0440\\u0435\\u043a\\u0442\\u043e\\u0440 \\u043f\\u043e IT', u'1.9': u'Web \\u0438\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440', u'12.92': u'\\u0418\\u043d\\u0444\\u043e\\u0440\\u043c\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u044b\\u0435 \\u0442\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438', u'6.254': u'\\u0420\\u0435\\u043a\\u0440\\u0443\\u0442\\u043c\\u0435\\u043d\\u0442', u'1.172': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'6.319': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b\\u043e\\u043c', u'15.320': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b\\u043e\\u043c', u'3.148': u'\\u041c\\u0435\\u043d\\u0435\\u0434\\u0436\\u0435\\u0440 \\u043f\\u043e \\u0440\\u0430\\u0431\\u043e\\u0442\\u0435 \\u0441 \\u043a\\u043b\\u0438\\u0435\\u043d\\u0442\\u0430\\u043c\\u0438', u'19.109': u'\\u041a\\u043e\\u043c\\u043f\\u043b\\u0435\\u043a\\u0441\\u043d\\u043e\\u0435 \\u0441\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u044e\\u0440\\u0438\\u0434\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u0445 \\u043b\\u0438\\u0446', u'19.108': u'\\u041a\\u043e\\u043c\\u043f\\u043b\\u0435\\u043a\\u0441\\u043d\\u043e\\u0435 \\u0441\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u0444\\u0438\\u0437\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u0445 \\u043b\\u0438\\u0446', u'15.281': u'\\u0421\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'17.112': u'\\u041a\\u043e\\u043c\\u043f\\u044c\\u044e\\u0442\\u0435\\u0440\\u043d\\u044b\\u0435 \\u043f\\u0440\\u043e\\u0433\\u0440\\u0430\\u043c\\u043c\\u044b', u'1.30': u'\\u0410\\u0440\\u0442-\\u0434\\u0438\\u0440\\u0435\\u043a\\u0442\\u043e\\u0440', u'15.731': u'\\u0417\\u0430\\u043a\\u0443\\u043f\\u043a\\u0438', u'22.316': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u0433\\u043e\\u0441\\u0442\\u0438\\u043d\\u0438\\u0446\\u0430\\u043c\\u0438', u'13.537': u'\\u041c\\u0435\\u0434\\u0438\\u0446\\u0438\\u043d\\u0441\\u043a\\u0438\\u0439 \\u0441\\u043e\\u0432\\u0435\\u0442\\u043d\\u0438\\u043a', u'2.164': u'\\u041d\\u0430\\u043b\\u043e\\u0433\\u0438', u'29.546': u'\\u041a\\u0440\\u0430\\u0441\\u043d\\u043e\\u0434\\u0435\\u0440\\u0435\\u0432\\u0449\\u0438\\u043a', u'29.547': u'\\u041a\\u0443\\u0437\\u043d\\u0435\\u0446', u'2.249': u'\\u0420\\u0430\\u0441\\u0447\\u0435\\u0442 \\u0441\\u0435\\u0431\\u0435\\u0441\\u0442\\u043e\\u0438\\u043c\\u043e\\u0441\\u0442\\u0438', u'29.545': u'\\u041a\\u043e\\u043c\\u043f\\u043b\\u0435\\u043a\\u0442\\u043e\\u0432\\u0449\\u0438\\u043a, \\u0423\\u043a\\u043b\\u0430\\u0434\\u0447\\u0438\\u043a-\\u0443\\u043f\\u0430\\u043a\\u043e\\u0432\\u0449\\u0438\\u043a', u'29.542': u'\\u0414\\u043e\\u0440\\u043e\\u0436\\u043d\\u044b\\u0435 \\u0440\\u0430\\u0431\\u043e\\u0447\\u0438\\u0435', u'29.543': u'\\u0416\\u0435\\u0441\\u0442\\u044f\\u043d\\u0449\\u0438\\u043a', u'29.540': u'\\u0413\\u0430\\u0440\\u0434\\u0435\\u0440\\u043e\\u0431\\u0449\\u0438\\u043a', u'29.541': u'\\u0414\\u0432\\u043e\\u0440\\u043d\\u0438\\u043a, \\u0423\\u0431\\u043e\\u0440\\u0449\\u0438\\u043a', u'29.548': u'\\u041b\\u0438\\u0444\\u0442\\u0435\\u0440', u'29.549': u'\\u041c\\u0430\\u0448\\u0438\\u043d\\u0438\\u0441\\u0442 \\u043f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u0430', u'27.500': u'\\u041f\\u043e\\u0432\\u0430\\u0440', u'27.501': u'\\u0421\\u0438\\u0434\\u0435\\u043b\\u043a\\u0430', u'7.503': u'\\u0410\\u0432\\u0442\\u043e\\u0441\\u043b\\u0435\\u0441\\u0430\\u0440\\u044c', u'19.285': u'\\u0421\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u043e\\u0442\\u0432\\u0435\\u0442\\u0441\\u0442\\u0432\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u0438', u'10.80': u'\\u0418\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440', u'18.142': u'\\u041c\\u0430\\u0448\\u0438\\u043d\\u043e\\u0441\\u0442\\u0440\\u043e\\u0435\\u043d\\u0438\\u0435', u'21.176': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'2.425': u'\\u042d\\u043a\\u043e\\u043d\\u043e\\u043c\\u0438\\u0441\\u0442', u'26.439': u'\\u0421\\u0442\\u0440\\u043e\\u0438\\u0442\\u0435\\u043b\\u044c\\u043d\\u044b\\u0435 \\u043c\\u0430\\u0442\\u0435\\u0440\\u0438\\u0430\\u043b\\u044b', u'26.437': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u0437\\u0430\\u043a\\u0443\\u043f\\u043a\\u0430\\u043c\\u0438', u'10.393': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'13.398': u'\\u041a\\u043b\\u0438\\u043d\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u0435 \\u0438\\u0441\\u0441\\u043b\\u0435\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u044f', u'3.213': u'\\u041f\\u043e\\u043b\\u0438\\u0442\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u0439 PR', u'20.525': u'\\u0417\\u0435\\u043c\\u043b\\u0435\\u0443\\u0441\\u0442\\u0440\\u043e\\u0439\\u0441\\u0442\\u0432\\u043e', u'20.527': u'\\u041e\\u0442\\u043e\\u043f\\u043b\\u0435\\u043d\\u0438\\u0435, \\u0432\\u0435\\u043d\\u0442\\u0438\\u043b\\u044f\\u0446\\u0438\\u044f \\u0438 \\u043a\\u043e\\u043d\\u0434\\u0438\\u0446\\u0438\\u043e\\u043d\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'20.526': u'\\u041e\\u0446\\u0435\\u043d\\u043a\\u0430', u'5.262': u'\\u0420\\u0443\\u043a\\u043e\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e \\u0431\\u0443\\u0445\\u0433\\u0430\\u043b\\u0442\\u0435\\u0440\\u0438\\u0435\\u0439', u'17.242': u'\\u041f\\u0440\\u044f\\u043c\\u044b\\u0435 \\u043f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0438', u'20.528': u'\\u0412\\u043e\\u0434\\u043e\\u0441\\u043d\\u0430\\u0431\\u0436\\u0435\\u043d\\u0438\\u0435 \\u0438 \\u043a\\u0430\\u043d\\u0430\\u043b\\u0438\\u0437\\u0430\\u0446\\u0438\\u044f', u'20.445': u'\\u0416\\u041a\\u0425', u'20.63': u'\\u0414\\u0438\\u0437\\u0430\\u0439\\u043d/\\u041e\\u0444\\u043e\\u0440\\u043c\\u043b\\u0435\\u043d\\u0438\\u0435', u'11.160': u'\\u041c\\u0443\\u0437\\u044b\\u043a\\u0430', u'18.360': u'\\u042d\\u043b\\u0435\\u043a\\u0442\\u0440\\u043e\\u044d\\u043d\\u0435\\u0440\\u0433\\u0435\\u0442\\u0438\\u043a\\u0430', u'18.361': u'\\u042d\\u043d\\u0435\\u0440\\u0433\\u0435\\u0442\\u0438\\u043a \\u043f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u0430', u'2.337': u'\\u0423\\u0447\\u0435\\u0442 \\u0441\\u0447\\u0435\\u0442\\u043e\\u0432 \\u0438 \\u043f\\u043b\\u0430\\u0442\\u0435\\u0436\\u0435\\u0439', u'2.335': u'\\u0423\\u0447\\u0435\\u0442 \\u0437\\u0430\\u0440\\u0430\\u0431\\u043e\\u0442\\u043d\\u043e\\u0439 \\u043f\\u043b\\u0430\\u0442\\u044b', u'1.82': u'\\u0418\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440', u'4.271': u'\\u0421\\u0438\\u043d\\u0445\\u0440\\u043e\\u043d\\u043d\\u044b\\u0439 \\u043f\\u0435\\u0440\\u0435\\u0432\\u043e\\u0434', u'13.49': u'\\u0412\\u0435\\u0442\\u0435\\u0440\\u0438\\u043d\\u0430\\u0440\\u0438\\u044f', u'9.289': u'\\u0421\\u0442\\u0440\\u043e\\u0438\\u0442\\u0435\\u043b\\u044c\\u0441\\u0442\\u0432\\u043e, \\u041d\\u0435\\u0434\\u0432\\u0438\\u0436\\u0438\\u043c\\u043e\\u0441\\u0442\\u044c', u'11.240': u'\\u041f\\u0440\\u043e\\u0447\\u0435\\u0435', u'11.243': u'\\u0420\\u0430\\u0434\\u0438\\u043e', u'4.278': u'\\u0421\\u043e\\u0442\\u0440\\u0443\\u0434\\u043d\\u0438\\u043a call-\\u0446\\u0435\\u043d\\u0442\\u0440\\u0430', u'3.328': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0440\\u043e\\u0435\\u043a\\u0442\\u0430\\u043c\\u0438', u'17.333': u'\\u0423\\u0441\\u043b\\u0443\\u0433\\u0438 \\u0434\\u043b\\u044f \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441\\u0430', u'29.556': u'\\u0420\\u0430\\u0437\\u043d\\u043e\\u0440\\u0430\\u0431\\u043e\\u0447\\u0438\\u0439', u'17.334': u'\\u0423\\u0441\\u043b\\u0443\\u0433\\u0438 \\u0434\\u043b\\u044f \\u043d\\u0430\\u0441\\u0435\\u043b\\u0435\\u043d\\u0438\\u044f', u'4.374': u'\\u0410\\u0425\\u041e', u'17.231': u'\\u041f\\u0440\\u043e\\u0434\\u0443\\u043a\\u0442\\u044b \\u043f\\u0438\\u0442\\u0430\\u043d\\u0438\\u044f', u'5.460': u'\\u0410\\u0432\\u0442\\u043e\\u043a\\u0440\\u0435\\u0434\\u0438\\u0442\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'12.6': u'Knowledge management', u'5.41': u'\\u0411\\u0443\\u043c\\u0430\\u0433\\u0438 \\u0441 \\u0444\\u0438\\u043a\\u0441\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u043d\\u043e\\u0439 \\u0434\\u043e\\u0445\\u043e\\u0434\\u043d\\u043e\\u0441\\u0442\\u044c\\u044e (fixed Income)', u'17.144': u'\\u041c\\u0435\\u0434\\u0438\\u0446\\u0438\\u043d\\u0430, \\u0424\\u0430\\u0440\\u043c\\u0430\\u0446\\u0435\\u0432\\u0442\\u0438\\u043a\\u0430', u'5.42': u'\\u0411\\u0443\\u0445\\u0433\\u0430\\u043b\\u0442\\u0435\\u0440', u'5.45': u'\\u0412\\u0430\\u043b\\u044e\\u0442\\u043d\\u044b\\u0439 \\u043a\\u043e\\u043d\\u0442\\u0440\\u043e\\u043b\\u044c', u'4.456': u'\\u0412\\u0435\\u0447\\u0435\\u0440\\u043d\\u0438\\u0439 \\u0441\\u0435\\u043a\\u0440\\u0435\\u0442\\u0430\\u0440\\u044c', u'17.418': u'\\u0421\\u0435\\u043b\\u044c\\u0441\\u043a\\u043e\\u0435 \\u0445\\u043e\\u0437\\u044f\\u0439\\u0441\\u0442\\u0432\\u043e', u'17.149': u'\\u041c\\u0435\\u043d\\u0435\\u0434\\u0436\\u0435\\u0440 \\u043f\\u043e \\u0440\\u0430\\u0431\\u043e\\u0442\\u0435 \\u0441 \\u043a\\u043b\\u0438\\u0435\\u043d\\u0442\\u0430\\u043c\\u0438', u'29.162': u'\\u041d\\u0430\\u043b\\u0430\\u0434\\u0447\\u0438\\u043a', u'19.147': u'\\u041c\\u0435\\u0434\\u0438\\u0446\\u0438\\u043d\\u0441\\u043a\\u043e\\u0435 \\u0441\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'1.400': u'\\u041e\\u043f\\u0442\\u0438\\u043c\\u0438\\u0437\\u0430\\u0446\\u0438\\u044f \\u0441\\u0430\\u0439\\u0442\\u0430 (SEO)', u'23.266': u'\\u0421\\u0435\\u043c\\u0435\\u0439\\u043d\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'17.417': u'\\u0421\\u0442\\u0440\\u043e\\u0438\\u0442\\u0435\\u043b\\u044c\\u043d\\u044b\\u0435 \\u043c\\u0430\\u0442\\u0435\\u0440\\u0438\\u0430\\u043b\\u044b', u'18.404': u'\\u0421\\u0435\\u0440\\u0442\\u0438\\u0444\\u0438\\u043a\\u0430\\u0446\\u0438\\u044f', u'18.568': u'\\u041a\\u043e\\u043d\\u0441\\u0442\\u0440\\u0443\\u043a\\u0442\\u043e\\u0440', u'25.381': u'\\u0421\\u0435\\u0440\\u0432\\u0438\\u0441\\u043d\\u044b\\u0439 \\u0438\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440', u'25.382': u'\\u0420\\u0443\\u043a\\u043e\\u0432\\u043e\\u0434\\u0438\\u0442\\u0435\\u043b\\u044c \\u0441\\u0435\\u0440\\u0432\\u0438\\u0441\\u043d\\u043e\\u0433\\u043e \\u0446\\u0435\\u043d\\u0442\\u0440\\u0430', u'25.383': u'\\u041c\\u0435\\u043d\\u0435\\u0434\\u0436\\u0435\\u0440 \\u043f\\u043e \\u0441\\u0435\\u0440\\u0432\\u0438\\u0441\\u0443 - \\u0441\\u0435\\u0442\\u0435\\u0432\\u044b\\u0435 \\u0438 \\u0442\\u0435\\u043b\\u0435\\u043a\\u043e\\u043c\\u043c\\u0443\\u043d\\u0438\\u043a\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u044b\\u0435 \\u0442\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438', u'25.384': u'\\u041c\\u0435\\u043d\\u0435\\u0434\\u0436\\u0435\\u0440 \\u043f\\u043e \\u0441\\u0435\\u0440\\u0432\\u0438\\u0441\\u0443 - \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0435 \\u043e\\u0431\\u043e\\u0440\\u0443\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'25.385': u'\\u041c\\u0435\\u043d\\u0435\\u0434\\u0436\\u0435\\u0440 \\u043f\\u043e \\u0441\\u0435\\u0440\\u0432\\u0438\\u0441\\u0443 - \\u0442\\u0440\\u0430\\u043d\\u0441\\u043f\\u043e\\u0440\\u0442', u'25.386': u'\\u0418\\u043d\\u0441\\u0442\\u0430\\u043b\\u043b\\u044f\\u0446\\u0438\\u044f \\u0438 \\u043d\\u0430\\u0441\\u0442\\u0440\\u043e\\u0439\\u043a\\u0430 \\u043e\\u0431\\u043e\\u0440\\u0443\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u044f', u'17.520': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0432\\u0435\\u0446 \\u0432 \\u043c\\u0430\\u0433\\u0430\\u0437\\u0438\\u043d\\u0435', u'17.486': u'\\u0421\\u0430\\u043d\\u0442\\u0435\\u0445\\u043d\\u0438\\u043a\\u0430', u'17.487': u'\\u0411\\u044b\\u0442\\u043e\\u0432\\u0430\\u044f \\u0442\\u0435\\u0445\\u043d\\u0438\\u043a\\u0430', u'18.118': u'\\u041a\\u043e\\u043d\\u0442\\u0440\\u043e\\u043b\\u044c \\u043a\\u0430\\u0447\\u0435\\u0441\\u0442\\u0432\\u0430', u'18.354': u'\\u042d\\u043a\\u043e\\u043b\\u043e\\u0433', u'18.585': u'\\u0410\\u0442\\u043e\\u043c\\u043d\\u0430\\u044f \\u044d\\u043d\\u0435\\u0440\\u0433\\u0435\\u0442\\u0438\\u043a\\u0430', u'22.530': u'\\u041e\\u0444\\u043e\\u0440\\u043c\\u043b\\u0435\\u043d\\u0438\\u0435 \\u0432\\u0438\\u0437', u'22.531': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u0442\\u0443\\u0440\\u0438\\u0441\\u0442\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u043c \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441\\u043e\\u043c', u'2.200': u'\\u041e\\u0444\\u0444\\u0448\\u043e\\u0440\\u044b', u'5.34': u'\\u0410\\u0443\\u0434\\u0438\\u0442, \\u0412\\u043d\\u0443\\u0442\\u0440\\u0435\\u043d\\u043d\\u0438\\u0439 \\u043a\\u043e\\u043d\\u0442\\u0440\\u043e\\u043b\\u044c', u'23.314': u'\\u0423\\u0433\\u043e\\u043b\\u043e\\u0432\\u043d\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'23.311': u'\\u0422\\u0440\\u0443\\u0434\\u043e\\u0432\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'17.15': u'\\u0410\\u0432\\u0442\\u043e\\u043c\\u043e\\u0431\\u0438\\u043b\\u0438, \\u0417\\u0430\\u043f\\u0447\\u0430\\u0441\\u0442\\u0438', u'18.73': u'\\u0417\\u0430\\u043a\\u0443\\u043f\\u043a\\u0438 \\u0438 \\u0441\\u043d\\u0430\\u0431\\u0436\\u0435\\u043d\\u0438\\u0435', u'18.75': u'\\u0417\\u043e\\u043e\\u0442\\u0435\\u0445\\u043d\\u0438\\u043a', u'2.43': u'\\u0411\\u0443\\u0445\\u0433\\u0430\\u043b\\u0442\\u0435\\u0440', u'2.44': u'\\u0411\\u044e\\u0434\\u0436\\u0435\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u0438 \\u043f\\u043b\\u0430\\u043d\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'2.46': u'\\u0412\\u0430\\u043b\\u044e\\u0442\\u043d\\u044b\\u0439 \\u043a\\u043e\\u043d\\u0442\\u0440\\u043e\\u043b\\u044c', u'1.536': u'CRM \\u0441\\u0438\\u0441\\u0442\\u0435\\u043c\\u044b', u'18.265': u'\\u0421\\u0435\\u043b\\u044c\\u0445\\u043e\\u0437\\u043f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e', u'13.567': u'\\u0414\\u0435\\u0444\\u0435\\u043a\\u0442\\u043e\\u043b\\u043e\\u0433, \\u041b\\u043e\\u0433\\u043e\\u043f\\u0435\\u0434', u'17.279': u'\\u0421\\u0442\\u0430\\u043d\\u043a\\u0438, \\u0442\\u044f\\u0436\\u0435\\u043b\\u043e\\u0435 \\u043e\\u0431\\u043e\\u0440\\u0443\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'19.483': u'\\u041f\\u0435\\u0440\\u0435\\u0441\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'15.313': u'\\u0422\\u0443\\u0440\\u0438\\u0437\\u043c, \\u0413\\u043e\\u0441\\u0442\\u0438\\u043d\\u0438\\u0446\\u044b, \\u0420\\u0435\\u0441\\u0442\\u043e\\u0440\\u0430\\u043d\\u044b', u'13.447': u'\\u041f\\u0441\\u0438\\u0445\\u043e\\u043b\\u043e\\u0433\\u0438\\u044f', u'8.135': u'\\u041b\\u0438\\u0447\\u043d\\u0430\\u044f \\u0431\\u0435\\u0437\\u043e\\u043f\\u0430\\u0441\\u043d\\u043e\\u0441\\u0442\\u044c', u'22.223': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0430 \\u0442\\u0443\\u0440\\u0438\\u0441\\u0442\\u0438\\u0447\\u0435\\u0441\\u043a\\u0438\\u0445 \\u0443\\u0441\\u043b\\u0443\\u0433', u'16.569': u'\\u0410\\u0440\\u0445\\u0438\\u0432\\u0430\\u0440\\u0438\\u0443\\u0441', u'10.191': u'\\u041d\\u0435\\u0444\\u0442\\u044c', u'21.69': u'\\u0416\\u0435\\u043b\\u0435\\u0437\\u043d\\u043e\\u0434\\u043e\\u0440\\u043e\\u0436\\u043d\\u044b\\u0435 \\u043f\\u0435\\u0440\\u0435\\u0432\\u043e\\u0437\\u043a\\u0438', u'26.574': u'\\u0422\\u0435\\u043d\\u0434\\u0435\\u0440\\u044b', u'7.222': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0430', u'17.65': u'\\u0414\\u0438\\u043b\\u0435\\u0440\\u0441\\u043a\\u0438\\u0435 \\u0441\\u0435\\u0442\\u0438', u'17.66': u'\\u0414\\u0438\\u0441\\u0442\\u0440\\u0438\\u0431\\u0443\\u0446\\u0438\\u044f', u'11.347': u'\\u0424\\u043e\\u0442\\u043e\\u0433\\u0440\\u0430\\u0444\\u0438\\u044f', u'27.496': u'\\u0412\\u043e\\u0441\\u043f\\u0438\\u0442\\u0430\\u0442\\u0435\\u043b\\u044c, \\u0413\\u0443\\u0432\\u0435\\u0440\\u043d\\u0430\\u043d\\u0442\\u043a\\u0430, \\u041d\\u044f\\u043d\\u044f', u'27.497': u'\\u0434\\u043e\\u043c\\u0440\\u0430\\u0431\\u043e\\u0442\\u043d\\u0438\\u0446\\u0430/\\u0434\\u043e\\u043c\\u0440\\u0430\\u0431\\u043e\\u0442\\u043d\\u0438\\u043a, \\u0413\\u043e\\u0440\\u043d\\u0438\\u0447\\u043d\\u0430\\u044f', u'20.186': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c/\\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'20.189': u'\\u041d\\u0435\\u0436\\u0438\\u043b\\u044b\\u0435 \\u043f\\u043e\\u043c\\u0435\\u0449\\u0435\\u043d\\u0438\\u044f', u'19.28': u'\\u0410\\u043d\\u0434\\u0435\\u0440\\u0440\\u0430\\u0439\\u0442\\u0435\\u0440', u'27.498': u'\\u041f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0432\\u043e\\u0434\\u0438\\u0442\\u0435\\u043b\\u044c', u'27.499': u'\\u0421\\u0430\\u0434\\u043e\\u0432\\u043d\\u0438\\u043a', u'19.357': u'\\u042d\\u043a\\u0441\\u043f\\u0435\\u0440\\u0442-\\u043e\\u0446\\u0435\\u043d\\u0449\\u0438\\u043a', u'20.20': u'\\u0410\\u0433\\u0435\\u043d\\u0442', u'14.355': u'\\u042d\\u043a\\u043e\\u043d\\u043e\\u043c\\u0438\\u043a\\u0430, \\u041c\\u0435\\u043d\\u0435\\u0434\\u0436\\u043c\\u0435\\u043d\\u0442', u'1.211': u'\\u041f\\u043e\\u0434\\u0434\\u0435\\u0440\\u0436\\u043a\\u0430, Helpdesk', u'18.201': u'\\u041e\\u0445\\u0440\\u0430\\u043d\\u0430 \\u0442\\u0440\\u0443\\u0434\\u0430', u'13.131': u'\\u041b\\u0435\\u0447\\u0430\\u0449\\u0438\\u0439 \\u0432\\u0440\\u0430\\u0447', u'21.17': u'\\u0410\\u0432\\u0442\\u043e\\u043f\\u0435\\u0440\\u0435\\u0432\\u043e\\u0437\\u043a\\u0438', u'21.12': u'\\u0410\\u0432\\u0438\\u0430\\u043f\\u0435\\u0440\\u0435\\u0432\\u043e\\u0437\\u043a\\u0438', u'13.138': u'\\u041c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433', u'18.208': u'\\u041f\\u0438\\u0449\\u0435\\u0432\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'3.244': u'\\u0420\\u0430\\u0434\\u0438\\u043e \\u0440\\u0435\\u043a\\u043b\\u0430\\u043c\\u0430', u'4.332': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u044f\\u044e\\u0449\\u0438\\u0439 \\u043e\\u0444\\u0438\\u0441\\u043e\\u043c(\\u041effice manager)', u'5.424': u'\\u041f\\u0430\\u0435\\u0432\\u044b\\u0435 \\u0444\\u043e\\u043d\\u0434\\u044b', u'5.426': u'\\u042d\\u043a\\u043e\\u043d\\u043e\\u043c\\u0438\\u0441\\u0442', u'4.338': u'\\u0423\\u0447\\u0435\\u0442 \\u0442\\u043e\\u0432\\u0430\\u0440\\u043e\\u043e\\u0431\\u043e\\u0440\\u043e\\u0442\\u0430', u'9.452': u'\\u0421\\u0442\\u0440\\u0430\\u0445\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'9.238': u'\\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e, \\u0422\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u044f', u'26.473': u'\\u0421\\u0442\\u0430\\u043d\\u043a\\u0438, \\u0422\\u044f\\u0436\\u0435\\u043b\\u043e\\u0435 \\u043e\\u0431\\u043e\\u0440\\u0443\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'3.31': u'\\u0410\\u0440\\u0442 \\u0434\\u0438\\u0440\\u0435\\u043a\\u0442\\u043e\\u0440', u'9.78': u'\\u0418\\u043d\\u0432\\u0435\\u0441\\u0442\\u0438\\u0446\\u0438\\u0438', u'17.156': u'\\u041c\\u043d\\u043e\\u0433\\u043e\\u0443\\u0440\\u043e\\u0432\\u043d\\u0435\\u0432\\u044b\\u0439 \\u043c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433', u'5.210': u'\\u041f\\u043b\\u0430\\u0441\\u0442\\u0438\\u043a\\u043e\\u0432\\u044b\\u0435 \\u043a\\u0430\\u0440\\u0442\\u044b', u'5.215': u'\\u041f\\u043e\\u0440\\u0442\\u0444\\u0435\\u043b\\u044c\\u043d\\u044b\\u0435 \\u0438\\u043d\\u0432\\u0435\\u0441\\u0442\\u0438\\u0446\\u0438\\u0438', u'17.405': u'\\u0421\\u0435\\u0440\\u0442\\u0438\\u0444\\u0438\\u043a\\u0430\\u0446\\u0438\\u044f', u'5.219': u'\\u041f\\u0440\\u0438\\u0432\\u043b\\u0435\\u0447\\u0435\\u043d\\u0438\\u0435 \\u043a\\u043b\\u0438\\u0435\\u043d\\u0442\\u043e\\u0432', u'17.152': u'\\u041c\\u0435\\u0442\\u0430\\u043b\\u043b\\u043e\\u043f\\u0440\\u043e\\u043a\\u0430\\u0442', u'4.52': u'\\u0412\\u043e\\u0434\\u0438\\u0442\\u0435\\u043b\\u044c', u'11.293': u'\\u0422\\u0435\\u043b\\u0435\\u0432\\u0438\\u0434\\u0435\\u043d\\u0438\\u0435', u'17.196': u'\\u041e\\u043f\\u0442\\u043e\\u0432\\u0430\\u044f \\u0442\\u043e\\u0440\\u0433\\u043e\\u0432\\u043b\\u044f', u'18.488': u'\\u041c\\u0435\\u0442\\u0440\\u043e\\u043b\\u043e\\u0433', u'18.330': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u0446\\u0435\\u0445\\u043e\\u043c', u'9.345': u'\\u0424\\u0438\\u043d\\u0430\\u043d\\u0441\\u044b', u'23.422': u'\\u0410\\u0432\\u0442\\u043e\\u0440\\u0441\\u043a\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'5.457': u'\\u0420\\u0438\\u0441\\u043a\\u0438: \\u043e\\u043f\\u0435\\u0440\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u044b\\u0435', u'21.480': u'\\u041a\\u043e\\u043d\\u0442\\u0435\\u0439\\u043d\\u0435\\u0440\\u043d\\u044b\\u0435 \\u043f\\u0435\\u0440\\u0435\\u0432\\u043e\\u0437\\u043a\\u0438', u'5.103': u'\\u041a\\u0430\\u0441\\u0441\\u043e\\u0432\\u043e\\u0435 \\u043e\\u0431\\u0441\\u043b\\u0443\\u0436\\u0438\\u0432\\u0430\\u043d\\u0438\\u0435, \\u0438\\u043d\\u043a\\u0430\\u0441\\u0441\\u0430\\u0446\\u0438\\u044f', u'5.450': u'\\u041a\\u0440\\u0435\\u0434\\u0438\\u0442\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435 \\u043c\\u0430\\u043b\\u043e\\u0433\\u043e \\u0438 \\u0441\\u0440\\u0435\\u0434\\u043d\\u0435\\u0433\\u043e \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441\\u0430', u'5.101': u'\\u0422\\u0440\\u0435\\u0439\\u0434\\u0438\\u043d\\u0433, \\u0414\\u0438\\u043b\\u0438\\u043d\\u0433', u'20.484': u'\\u0413\\u0435\\u043e\\u0434\\u0435\\u0437\\u0438\\u044f \\u0438 \\u043a\\u0430\\u0440\\u0442\\u043e\\u0433\\u0440\\u0430\\u0444\\u0438\\u044f', u'21.563': u'\\u041a\\u043b\\u0430\\u0434\\u043e\\u0432\\u0449\\u0438\\u043a', u'5.459': u'\\u0420\\u0438\\u0441\\u043a\\u0438: \\u0440\\u044b\\u043d\\u043e\\u0447\\u043d\\u044b\\u0435', u'5.458': u'\\u0420\\u0438\\u0441\\u043a\\u0438: \\u0444\\u0438\\u043d\\u0430\\u043d\\u0441\\u043e\\u0432\\u044b\\u0435', u'15.288': u'\\u0421\\u0442\\u0440\\u043e\\u0438\\u0442\\u0435\\u043b\\u044c\\u0441\\u0442\\u0432\\u043e, \\u0410\\u0440\\u0445\\u0438\\u0442\\u0435\\u043a\\u0442\\u0443\\u0440\\u0430', u'5.79': u'\\u0418\\u043d\\u0432\\u0435\\u0441\\u0442\\u0438\\u0446\\u0438\\u043e\\u043d\\u043d\\u0430\\u044f \\u043a\\u043e\\u043c\\u043f\\u0430\\u043d\\u0438\\u044f', u'18.130': u'\\u0414\\u0435\\u0440\\u0435\\u0432\\u043e\\u043e\\u0431\\u0440\\u0430\\u0431\\u043e\\u0442\\u043a\\u0430, \\u041b\\u0435\\u0441\\u043d\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'23.214': u'\\u041f\\u043e\\u043c\\u043e\\u0449\\u043d\\u0438\\u043a', u'17.580': u'\\u041a\\u043b\\u0438\\u043d\\u0438\\u043d\\u0433\\u043e\\u0432\\u044b\\u0435 \\u0443\\u0441\\u043b\\u0443\\u0433\\u0438', u'27.584': u'\\u0420\\u0435\\u043f\\u0435\\u0442\\u0438\\u0442\\u043e\\u0440', u'23.352': u'\\u0426\\u0435\\u043d\\u043d\\u044b\\u0435 \\u0431\\u0443\\u043c\\u0430\\u0433\\u0438, \\u0420\\u044b\\u043d\\u043a\\u0438 \\u043a\\u0430\\u043f\\u0438\\u0442\\u0430\\u043b\\u0430', u'6.309': u'\\u0422\\u0440\\u0435\\u043d\\u0438\\u043d\\u0433\\u0438', u'18.348': u'\\u0425\\u0438\\u043c\\u0438\\u0447\\u0435\\u0441\\u043a\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'19.170': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'3.150': u'\\u041c\\u0435\\u043d\\u0435\\u0434\\u0436\\u043c\\u0435\\u043d\\u0442 \\u043f\\u0440\\u043e\\u0434\\u0443\\u043a\\u0442\\u0430 (Product manager)', u'3.151': u'\\u041c\\u0435\\u0440\\u0447\\u0435\\u043d\\u0434\\u0430\\u0439\\u0437\\u0438\\u043d\\u0433', u'1.25': u'\\u0410\\u043d\\u0430\\u043b\\u0438\\u0442\\u0438\\u043a', u'18.431': u'\\u0413\\u043b\\u0430\\u0432\\u043d\\u044b\\u0439 \\u043c\\u0435\\u0445\\u0430\\u043d\\u0438\\u043a', u'3.294': u'\\u0422\\u0435\\u043b\\u0435\\u0432\\u0438\\u0437\\u0438\\u043e\\u043d\\u043d\\u0430\\u044f \\u0440\\u0435\\u043a\\u043b\\u0430\\u043c\\u0430', u'29.555': u'\\u041f\\u0435\\u0440\\u0435\\u043c\\u043e\\u0442\\u0447\\u0438\\u043a', u'29.554': u'\\u041f\\u0430\\u0441\\u0442\\u0443\\u0445, \\u0427\\u0430\\u0431\\u0430\\u043d', u'29.557': u'\\u0421\\u0430\\u043d\\u0442\\u0435\\u0445\\u043d\\u0438\\u043a', u'7.502': u'\\u0422\\u043e\\u043d\\u0438\\u0440\\u043e\\u0432\\u0449\\u0438\\u043a', u'29.551': u'\\u041c\\u0430\\u0448\\u0438\\u043d\\u0438\\u0441\\u0442 \\u044d\\u043a\\u0441\\u043a\\u0430\\u0432\\u0430\\u0442\\u043e\\u0440\\u0430', u'29.550': u'\\u041c\\u0430\\u0448\\u0438\\u043d\\u0438\\u0441\\u0442 \\u0441\\u0446\\u0435\\u043d\\u044b', u'29.553': u'\\u041e\\u043f\\u0435\\u0440\\u0430\\u0442\\u043e\\u0440 \\u0441\\u0442\\u0430\\u043d\\u043a\\u043e\\u0432', u'29.552': u'\\u041c\\u0435\\u0445\\u0430\\u043d\\u0438\\u043a', u'9.168': u'\\u041d\\u0430\\u0443\\u043a\\u0430, \\u041e\\u0431\\u0440\\u0430\\u0437\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'15.68': u'\\u0414\\u043e\\u0431\\u044b\\u0447\\u0430 \\u0441\\u044b\\u0440\\u044c\\u044f', u'29.559': u'\\u0428\\u0442\\u0443\\u043a\\u0430\\u0442\\u0443\\u0440', u'29.558': u'\\u0428\\u043b\\u0438\\u0444\\u043e\\u0432\\u0449\\u0438\\u043a', u'7.267': u'\\u0421\\u0435\\u0440\\u0432\\u0438\\u0441\\u043d\\u043e\\u0435 \\u043e\\u0431\\u0441\\u043b\\u0443\\u0436\\u0438\\u0432\\u0430\\u043d\\u0438\\u0435', u'17.24': u'\\u0410\\u043b\\u043a\\u043e\\u0433\\u043e\\u043b\\u044c', u'8.519': u'\\u0418\\u043d\\u043a\\u0430\\u0441\\u0441\\u0430\\u0442\\u043e\\u0440', u'13.185': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'15.346': u'\\u0424\\u0438\\u043d\\u0430\\u043d\\u0441\\u044b, \\u0411\\u0430\\u043d\\u043a\\u0438, \\u0418\\u043d\\u0432\\u0435\\u0441\\u0442\\u0438\\u0446\\u0438\\u0438', u'18.153': u'\\u041c\\u0435\\u0442\\u0430\\u043b\\u043b\\u0443\\u0440\\u0433\\u0438\\u044f', u'26.449': u'\\u042d\\u043b\\u0435\\u043a\\u0442\\u0440\\u043e\\u0442\\u0435\\u0445\\u043d\\u0438\\u0447\\u0435\\u0441\\u043a\\u043e\\u0435 \\u043e\\u0431\\u043e\\u0440\\u0443\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435/\\u0441\\u0432\\u0435\\u0442\\u043e\\u0442\\u0435\\u0445\\u043d\\u0438\\u043a\\u0430', u'26.408': u'\\u0410\\u043b\\u043a\\u043e\\u0433\\u043e\\u043b\\u044c', u'26.409': u'\\u0413\\u0421\\u041c, \\u043d\\u0435\\u0444\\u0442\\u044c, \\u0431\\u0435\\u043d\\u0437\\u0438\\u043d', u'2.434': u'\\u041f\\u043b\\u0430\\u043d\\u043e\\u0432\\u043e-\\u044d\\u043a\\u043e\\u043d\\u043e\\u043c\\u0438\\u0447\\u0435\\u0441\\u043a\\u043e\\u0435 \\u0443\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435', u'3.64': u'\\u0414\\u0438\\u0437\\u0430\\u0439\\u043d\\u0435\\u0440', u'8.260': u'\\u0420\\u0443\\u043a\\u043e\\u0432\\u043e\\u0434\\u0438\\u0442\\u0435\\u043b\\u044c \\u0421\\u0411', u'26.406': u'\\u0421\\u0435\\u0440\\u0442\\u0438\\u0444\\u0438\\u043a\\u0430\\u0446\\u0438\\u044f', u'26.407': u'\\u0410\\u0432\\u0442\\u043e\\u043c\\u043e\\u0431\\u0438\\u043b\\u0438, \\u0417\\u0430\\u043f\\u0447\\u0430\\u0441\\u0442\\u0438', u'10.54': u'\\u0413\\u0435\\u043e\\u043b\\u043e\\u0433\\u043e\\u0440\\u0430\\u0437\\u0432\\u0435\\u0434\\u043a\\u0430', u'21.53': u'\\u0412\\u042d\\u0414', u'3.209': u'\\u041f\\u043b\\u0430\\u043d\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435, \\u0420\\u0430\\u0437\\u043c\\u0435\\u0449\\u0435\\u043d\\u0438\\u0435 \\u0440\\u0435\\u043a\\u043b\\u0430\\u043c\\u044b', u'20.325': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0440\\u043e\\u0435\\u043a\\u0442\\u0430\\u043c\\u0438', u'12.251': u'\\u0420\\u0435\\u0438\\u043d\\u0436\\u0438\\u043d\\u0438\\u0440\\u0438\\u043d\\u0433 \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441 \\u043f\\u0440\\u043e\\u0446\\u0435\\u0441\\u0441\\u043e\\u0432', u'12.252': u'\\u0420\\u0435\\u0438\\u043d\\u0436\\u0438\\u043d\\u0438\\u0440\\u0438\\u043d\\u0433, \\u0410\\u0443\\u0442\\u0441\\u043e\\u0440\\u0441\\u0438\\u043d\\u0433 \\u0444\\u0438\\u043d\\u0430\\u043d\\u0441\\u043e\\u0432\\u043e\\u0439 \\u0444\\u0443\\u043d\\u043a\\u0446\\u0438\\u0438', u'3.206': u'\\u041f\\u0435\\u0447\\u0430\\u0442\\u043d\\u0430\\u044f \\u0440\\u0435\\u043a\\u043b\\u0430\\u043c\\u0430', u'20.287': u'\\u0421\\u0442\\u0440\\u043e\\u0438\\u0442\\u0435\\u043b\\u044c\\u0441\\u0442\\u0432\\u043e', u'8.356': u'\\u042d\\u043a\\u043e\\u043d\\u043e\\u043c\\u0438\\u0447\\u0435\\u0441\\u043a\\u0430\\u044f \\u0438 \\u0438\\u043d\\u0444\\u043e\\u0440\\u043c\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u0430\\u044f \\u0431\\u0435\\u0437\\u043e\\u043f\\u0430\\u0441\\u043d\\u043e\\u0441\\u0442\\u044c', u'9.562': u'\\u0410\\u043d\\u0442\\u0438\\u043a\\u0440\\u0438\\u0437\\u0438\\u0441\\u043d\\u043e\\u0435 \\u0443\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435', u'1.246': u'\\u0420\\u0430\\u0437\\u0432\\u0438\\u0442\\u0438\\u0435 \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441\\u0430', u'17.397': u'\\u042d\\u043b\\u0435\\u043a\\u0442\\u0440\\u043e\\u0442\\u0435\\u0445\\u043d\\u0438\\u0447\\u0435\\u0441\\u043a\\u043e\\u0435 \\u043e\\u0431\\u043e\\u0440\\u0443\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435, \\u0421\\u0432\\u0435\\u0442\\u043e\\u0442\\u0435\\u0445\\u043d\\u0438\\u043a\\u0430', u'1.327': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0440\\u043e\\u0435\\u043a\\u0442\\u0430\\u043c\\u0438', u'18.245': u'\\u0420\\u0430\\u0434\\u0438\\u043e\\u044d\\u043b\\u0435\\u043a\\u0442\\u0440\\u043e\\u043d\\u043d\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'10.470': u'\\u0411\\u0443\\u0440\\u0435\\u043d\\u0438\\u0435', u'10.471': u'\\u041c\\u0430\\u0440\\u043a\\u0448\\u0435\\u0439\\u0434\\u0435\\u0440', u'18.373': u'\\u0421\\u0443\\u0434\\u043e\\u0441\\u0442\\u0440\\u043e\\u0435\\u043d\\u0438\\u0435', u'3.1': u'Below The Line (BTL)', u'11.157': u'\\u041c\\u043e\\u0434\\u0430', u'20.70': u'\\u0416\\u0438\\u043b\\u044c\\u0435', u'3.8': u'PR, \\u041c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433\\u043e\\u0432\\u044b\\u0435 \\u043a\\u043e\\u043c\\u043c\\u0443\\u043d\\u0438\\u043a\\u0430\\u0446\\u0438\\u0438', u'22.39': u'\\u0411\\u0440\\u043e\\u043d\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'4.205': u'\\u041f\\u0435\\u0440\\u0441\\u043e\\u043d\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0430\\u0441\\u0441\\u0438\\u0441\\u0442\\u0435\\u043d\\u0442', u'4.207': u'\\u041f\\u0438\\u0441\\u044c\\u043c\\u0435\\u043d\\u043d\\u044b\\u0439 \\u043f\\u0435\\u0440\\u0435\\u0432\\u043e\\u0434', u'5.371': u'\\u041a\\u0440\\u0435\\u0434\\u0438\\u0442\\u044b: \\u0440\\u043e\\u0437\\u043d\\u0438\\u0447\\u043d\\u044b\\u0435', u'22.35': u'\\u0411\\u0430\\u043d\\u043a\\u0435\\u0442\\u044b', u'3.507': u'\\u0422\\u0430\\u0439\\u043d\\u044b\\u0439 \\u043f\\u043e\\u043a\\u0443\\u043f\\u0430\\u0442\\u0435\\u043b\\u044c', u'17.324': u'\\u0423\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u0435 \\u043f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0430\\u043c\\u0438', u'3.509': u'\\u041f\\u0440\\u043e\\u043c\\u043e\\u0443\\u0442\\u0435\\u0440', u'3.508': u'\\u041f\\u0440\\u043e\\u0432\\u0435\\u0434\\u0435\\u043d\\u0438\\u0435 \\u043e\\u043f\\u0440\\u043e\\u0441\\u043e\\u0432, \\u0418\\u043d\\u0442\\u0435\\u0440\\u0432\\u044c\\u044e\\u0435\\u0440', u'9.307': u'\\u0422\\u0440\\u0430\\u043d\\u0441\\u043f\\u043e\\u0440\\u0442, \\u041b\\u043e\\u0433\\u0438\\u0441\\u0442\\u0438\\u043a\\u0430', u'4.429': u'\\u0414\\u0435\\u043b\\u043e\\u043f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e', u'4.428': u'\\u041f\\u043e\\u0441\\u043b\\u0435\\u0434\\u043e\\u0432\\u0430\\u0442\\u0435\\u043b\\u044c\\u043d\\u044b\\u0439 \\u043f\\u0435\\u0440\\u0435\\u0432\\u043e\\u0434', u'15.390': u'\\u0410\\u0432\\u0442\\u043e\\u043c\\u043e\\u0431\\u0438\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441', u'15.391': u'\\u041a\\u043e\\u043d\\u0441\\u0443\\u043b\\u044c\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'23.159': u'\\u041c\\u043e\\u0440\\u0441\\u043a\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'3.119': u'\\u041a\\u043e\\u043f\\u0438\\u0440\\u0430\\u0439\\u0442\\u0435\\u0440', u'9.115': u'\\u041a\\u043e\\u043d\\u0441\\u0443\\u043b\\u044c\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'29.544': u'\\u0417\\u0430\\u043f\\u0440\\u0430\\u0432\\u0449\\u0438\\u043a \\u043a\\u0430\\u0440\\u0442\\u0440\\u0438\\u0434\\u0436\\u0435\\u0439', u'5.257': u'\\u0420\\u043e\\u0437\\u043d\\u0438\\u0447\\u043d\\u044b\\u0439 \\u0431\\u0438\\u0437\\u043d\\u0435\\u0441', u'5.106': u'\\u041a\\u043e\\u043c\\u043c\\u0435\\u0440\\u0447\\u0435\\u0441\\u043a\\u0438\\u0439 \\u0431\\u0430\\u043d\\u043a', u'15.140': u'\\u041c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433, \\u0420\\u0435\\u043a\\u043b\\u0430\\u043c\\u0430, PR', u'23.2': u'Compliance', u'5.250': u'\\u0420\\u0430\\u0441\\u0447\\u0435\\u0442\\u044b', u'5.394': u'\\u042d\\u043c\\u0438\\u0441\\u0441\\u0438\\u0438', u'14.169': u'\\u041d\\u0430\\u0443\\u043a\\u0438 \\u043e \\u0417\\u0435\\u043c\\u043b\\u0435', u'17.625': u'\\u0420\\u0435\\u0448\\u0435\\u043d\\u0438\\u044f \\u043f\\u043e \\u0430\\u0432\\u0442\\u043e\\u043c\\u0430\\u0442\\u0438\\u0437\\u0430\\u0446\\u0438\\u0438 \\u043f\\u0440\\u043e\\u0446\\u0435\\u0441\\u0441\\u043e\\u0432', u'21.158': u'\\u041c\\u043e\\u0440\\u0441\\u043a\\u0438\\u0435/\\u0420\\u0435\\u0447\\u043d\\u044b\\u0435 \\u043f\\u0435\\u0440\\u0435\\u0432\\u043e\\u0437\\u043a\\u0438', u'20.387': u'\\u041f\\u0440\\u043e\\u0440\\u0430\\u0431', u'22.248': u'\\u0420\\u0430\\u0437\\u043c\\u0435\\u0449\\u0435\\u043d\\u0438\\u0435, \\u041e\\u0431\\u0441\\u043b\\u0443\\u0436\\u0438\\u0432\\u0430\\u043d\\u0438\\u0435 \\u0433\\u043e\\u0441\\u0442\\u0435\\u0439', u'14.364': u'\\u042f\\u0437\\u044b\\u043a\\u0438', u'5.27': u'\\u0410\\u043d\\u0430\\u043b\\u0438\\u0442\\u0438\\u043a', u'9.94': u'\\u0418\\u043d\\u0444\\u043e\\u0440\\u043c\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u044b\\u0435 \\u0442\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438, \\u0418\\u043d\\u0442\\u0435\\u0440\\u043d\\u0435\\u0442, \\u041c\\u0443\\u043b\\u044c\\u0442\\u0438\\u043c\\u0435\\u0434\\u0438\\u0430', u'9.95': u'\\u0418\\u0441\\u043a\\u0443\\u0441\\u0441\\u0442\\u0432\\u043e, \\u0420\\u0430\\u0437\\u0432\\u043b\\u0435\\u0447\\u0435\\u043d\\u0438\\u044f, \\u041c\\u0430\\u0441\\u0441-\\u043c\\u0435\\u0434\\u0438\\u0430', u'5.23': u'\\u0410\\u043a\\u0446\\u0438\\u0438, \\u0426\\u0435\\u043d\\u043d\\u044b\\u0435 \\u0431\\u0443\\u043c\\u0430\\u0433\\u0438', u'29.511': u'\\u0422\\u043e\\u043a\\u0430\\u0440\\u044c, \\u0424\\u0440\\u0435\\u0437\\u0435\\u0440\\u043e\\u0432\\u0449\\u0438\\u043a', u'29.510': u'\\u0421\\u043b\\u0435\\u0441\\u0430\\u0440\\u044c', u'1.116': u'\\u041a\\u043e\\u043d\\u0442\\u0435\\u043d\\u0442', u'1.117': u'\\u0422\\u0435\\u0441\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'1.110': u'\\u041a\\u043e\\u043c\\u043f\\u044c\\u044e\\u0442\\u0435\\u0440\\u043d\\u0430\\u044f \\u0431\\u0435\\u0437\\u043e\\u043f\\u0430\\u0441\\u043d\\u043e\\u0441\\u0442\\u044c', u'29.514': u'\\u0421\\u0432\\u0430\\u0440\\u0449\\u0438\\u043a', u'29.517': u'\\u0421\\u0431\\u043e\\u0440\\u0449\\u0438\\u043a', u'1.113': u'\\u041a\\u043e\\u043d\\u0441\\u0430\\u043b\\u0442\\u0438\\u043d\\u0433, \\u0410\\u0443\\u0442\\u0441\\u043e\\u0440\\u0441\\u0438\\u043d\\u0433', u'18.298': u'\\u0422\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433, \\u041c\\u044f\\u0441\\u043e- \\u0438 \\u043f\\u0442\\u0438\\u0446\\u0435\\u043f\\u0435\\u0440\\u0435\\u0440\\u0430\\u0431\\u043e\\u0442\\u043a\\u0430', u'29.582': u'\\u041f\\u0440\\u043e\\u0432\\u043e\\u0434\\u043d\\u0438\\u043a', u'29.583': u'\\u041c\\u0430\\u043b\\u044f\\u0440', u'1.50': u'\\u0421\\u0438\\u0441\\u0442\\u0435\\u043c\\u044b \\u0443\\u043f\\u0440\\u0430\\u0432\\u043b\\u0435\\u043d\\u0438\\u044f \\u043f\\u0440\\u0435\\u0434\\u043f\\u0440\\u0438\\u044f\\u0442\\u0438\\u0435\\u043c (ERP)', u'29.581': u'\\u041c\\u043e\\u043d\\u0442\\u0430\\u0436\\u043d\\u0438\\u043a', u'15.96': u'\\u0418\\u0441\\u043a\\u0443\\u0441\\u0441\\u0442\\u0432\\u043e, \\u0420\\u0430\\u0437\\u0432\\u043b\\u0435\\u0447\\u0435\\u043d\\u0438\\u044f, \\u041c\\u0430\\u0441\\u0441-\\u043c\\u0435\\u0434\\u0438\\u0430', u'22.504': u'\\u0425\\u043e\\u0441\\u0442\\u0435\\u0441', u'29.588': u'\\u0414\\u0440\\u0443\\u0433\\u043e\\u0435', u'17.256': u'\\u0420\\u043e\\u0437\\u043d\\u0438\\u0447\\u043d\\u0430\\u044f \\u0442\\u043e\\u0440\\u0433\\u043e\\u0432\\u043b\\u044f', u'2.100': u'\\u041a\\u0430\\u0437\\u043d\\u0430\\u0447\\u0435\\u0439\\u0441\\u0442\\u0432\\u043e', u'2.102': u'\\u041a\\u0430\\u0441\\u0441\\u0438\\u0440, \\u0418\\u043d\\u043a\\u0430\\u0441\\u0441\\u0430\\u0442\\u043e\\u0440', u'16.571': u'\\u0411\\u0438\\u0431\\u043b\\u0438\\u043e\\u0442\\u0435\\u043a\\u0430\\u0440\\u044c', u'16.570': u'\\u0410\\u0442\\u0442\\u0430\\u0448\\u0435', u'13.228': u'\\u041b\\u0435\\u043a\\u0430\\u0440\\u0441\\u0442\\u0432\\u0435\\u043d\\u043d\\u044b\\u0435 \\u043f\\u0440\\u0435\\u043f\\u0430\\u0440\\u0430\\u0442\\u044b', u'13.229': u'\\u041c\\u0435\\u0434\\u0438\\u0446\\u0438\\u043d\\u0441\\u043a\\u043e\\u0435 \\u043e\\u0431\\u043e\\u0440\\u0443\\u0434\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'13.220': u'\\u041f\\u0440\\u043e\\u0432\\u0438\\u0437\\u043e\\u0440', u'12.122': u'\\u041a\\u043e\\u0440\\u043f\\u043e\\u0440\\u0430\\u0442\\u0438\\u0432\\u043d\\u044b\\u0435 \\u0444\\u0438\\u043d\\u0430\\u043d\\u0441\\u044b', u'7.392': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c /\\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430 ', u'2.261': u'\\u0420\\u0443\\u043a\\u043e\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e \\u0431\\u0443\\u0445\\u0433\\u0430\\u043b\\u0442\\u0435\\u0440\\u0438\\u0435\\u0439', u'13.227': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0438', u'9.521': u'\\u0421\\u043f\\u043e\\u0440\\u0442\\u0438\\u0432\\u043d\\u044b\\u0435 \\u043a\\u043b\\u0443\\u0431\\u044b, \\u0424\\u0438\\u0442\\u043d\\u0435\\u0441, \\u0421\\u0430\\u043b\\u043e\\u043d\\u044b \\u043a\\u0440\\u0430\\u0441\\u043e\\u0442\\u044b', u'18.291': u'\\u0422\\u0430\\u0431\\u0430\\u0447\\u043d\\u0430\\u044f \\u043f\\u0440\\u043e\\u043c\\u044b\\u0448\\u043b\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u044c', u'9.139': u'\\u041c\\u0430\\u0440\\u043a\\u0435\\u0442\\u0438\\u043d\\u0433, \\u0420\\u0435\\u043a\\u043b\\u0430\\u043c\\u0430, PR', u'15.237': u'\\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e, \\u0422\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438', u'8.461': u'\\u0421\\u0438\\u0441\\u0442\\u0435\\u043c\\u044b \\u0432\\u0438\\u0434\\u0435\\u043e\\u043d\\u0430\\u0431\\u043b\\u044e\\u0434\\u0435\\u043d\\u0438\\u044f', u'8.462': u'\\u0412\\u0437\\u044b\\u0441\\u043a\\u0430\\u043d\\u0438\\u0435 \\u0437\\u0430\\u0434\\u043e\\u043b\\u0436\\u0435\\u043d\\u043d\\u043e\\u0441\\u0442\\u0438, \\u041a\\u043e\\u043b\\u043b\\u0435\\u043a\\u0442\\u043e\\u0440\\u0441\\u043a\\u0430\\u044f \\u0434\\u0435\\u044f\\u0442\\u0435\\u043b\\u044c\\u043d\\u043e\\u0441\\u0442\\u044c', u'22.529': u'\\u0410\\u043d\\u0438\\u043c\\u0430\\u0446\\u0438\\u044f', u'16.194': u'\\u041e\\u0431\\u0449\\u0435\\u0441\\u0442\\u0432\\u0435\\u043d\\u043d\\u044b\\u0435 \\u043e\\u0440\\u0433\\u0430\\u043d\\u0438\\u0437\\u0430\\u0446\\u0438\\u0438', u'17.358': u'\\u042d\\u043b\\u0435\\u043a\\u0442\\u0440\\u043e\\u043d\\u0438\\u043a\\u0430, \\u0444\\u043e\\u0442\\u043e, \\u0432\\u0438\\u0434\\u0435\\u043e', u'14.91': u'\\u0418\\u043d\\u0444\\u043e\\u0440\\u043c\\u0430\\u0442\\u0438\\u043a\\u0430, \\u0418\\u043d\\u0444\\u043e\\u0440\\u043c\\u0430\\u0446\\u0438\\u043e\\u043d\\u043d\\u044b\\u0435 \\u0441\\u0438\\u0441\\u0442\\u0435\\u043c\\u044b', u'17.350': u'\\u0426\\u0432\\u0435\\u0442\\u043d\\u044b\\u0435 \\u043c\\u0435\\u0442\\u0430\\u043b\\u043b\\u044b', u'1.203': u'\\u041f\\u0435\\u0440\\u0435\\u0434\\u0430\\u0447\\u0430 \\u0434\\u0430\\u043d\\u043d\\u044b\\u0445 \\u0438 \\u0434\\u043e\\u0441\\u0442\\u0443\\u043f \\u0432 \\u0438\\u043d\\u0442\\u0435\\u0440\\u043d\\u0435\\u0442', u'11.218': u'\\u041f\\u0440\\u0435\\u0441\\u0441\\u0430', u'19.586': u'\\u0412\\u0440\\u0430\\u0447-\\u044d\\u043a\\u0441\\u043f\\u0435\\u0440\\u0442', u'13.128': u'\\u041b\\u0430\\u0431\\u043e\\u0440\\u0430\\u043d\\u0442', u'12.280': u'\\u0421\\u0442\\u0440\\u0430\\u0442\\u0435\\u0433\\u0438\\u044f', u'6.184': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'21.403': u'\\u0411\\u0438\\u0437\\u043d\\u0435\\u0441-\\u0430\\u0432\\u0438\\u0430\\u0446\\u0438\\u044f', u'21.402': u'\\u0413\\u0440\\u0430\\u0436\\u0434\\u0430\\u043d\\u0441\\u043a\\u0430\\u044f \\u0430\\u0432\\u0438\\u0430\\u0446\\u0438\\u044f', u'3.423': u'\\u0410\\u0441\\u0441\\u0438\\u0441\\u0442\\u0435\\u043d\\u0442', u'3.26': u'\\u0410\\u043d\\u0430\\u043b\\u0438\\u0442\\u0438\\u043a', u'14.217': u'\\u041f\\u0440\\u0435\\u043f\\u043e\\u0434\\u0430\\u0432\\u0430\\u043d\\u0438\\u0435', u'9.226': u'\\u041f\\u0440\\u043e\\u0434\\u0430\\u0436\\u0438', u'18.143': u'\\u041c\\u0435\\u0431\\u0435\\u043b\\u044c\\u043d\\u043e\\u0435 \\u043f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e', u'5.163': u'\\u041d\\u0430\\u043b\\u043e\\u0433\\u0438', u'23.442': u'\\u041c\\u0435\\u0436\\u0434\\u0443\\u043d\\u0430\\u0440\\u043e\\u0434\\u043d\\u043e\\u0435 \\u043f\\u0440\\u0430\\u0432\\u043e', u'1.475': u'\\u0418\\u0433\\u0440\\u043e\\u0432\\u043e\\u0435 \\u041f\\u041e', u'1.474': u'\\u0421\\u0442\\u0430\\u0440\\u0442\\u0430\\u043f\\u044b', u'1.274': u'\\u0421\\u0438\\u0441\\u0442\\u0435\\u043c\\u044b \\u0430\\u0432\\u0442\\u043e\\u043c\\u0430\\u0442\\u0438\\u0437\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u043d\\u043e\\u0433\\u043e \\u043f\\u0440\\u043e\\u0435\\u043a\\u0442\\u0438\\u0440\\u043e\\u0432\\u0430\\u043d\\u0438\\u044f', u'1.277': u'\\u0421\\u043e\\u0442\\u043e\\u0432\\u044b\\u0435, \\u0411\\u0435\\u0441\\u043f\\u0440\\u043e\\u0432\\u043e\\u0434\\u043d\\u044b\\u0435 \\u0442\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438', u'1.270': u'\\u0421\\u0435\\u0442\\u0435\\u0432\\u044b\\u0435 \\u0442\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433\\u0438\\u0438', u'1.273': u'\\u0421\\u0438\\u0441\\u0442\\u0435\\u043c\\u043d\\u044b\\u0439 \\u0430\\u0434\\u043c\\u0438\\u043d\\u0438\\u0441\\u0442\\u0440\\u0430\\u0442\\u043e\\u0440', u'1.272': u'\\u0421\\u0438\\u0441\\u0442\\u0435\\u043c\\u043d\\u0430\\u044f \\u0438\\u043d\\u0442\\u0435\\u0433\\u0440\\u0430\\u0446\\u0438\\u044f', u'17.440': u'\\u0422\\u0435\\u043a\\u0441\\u0442\\u0438\\u043b\\u044c, \\u041e\\u0434\\u0435\\u0436\\u0434\\u0430, \\u041e\\u0431\\u0443\\u0432\\u044c', u'17.183': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430', u'23.187': u'\\u041d\\u0435\\u0434\\u0432\\u0438\\u0436\\u0438\\u043c\\u043e\\u0441\\u0442\\u044c', u'18.300': u'\\u0422\\u0435\\u0445\\u043d\\u043e\\u043b\\u043e\\u0433, \\u041f\\u0440\\u043e\\u0438\\u0437\\u0432\\u043e\\u0434\\u0441\\u0442\\u0432\\u043e \\u0441\\u0430\\u0445\\u0430\\u0440\\u0430', u'21.275': u'\\u0421\\u043a\\u043b\\u0430\\u0434\\u0441\\u043a\\u043e\\u0435 \\u0445\\u043e\\u0437\\u044f\\u0439\\u0441\\u0442\\u0432\\u043e', u'4.494': u'\\u0423\\u0431\\u043e\\u0440\\u0449\\u0438\\u0446\\u0430', u'5.444': u'\\u0424\\u0438\\u043d\\u0430\\u043d\\u0441\\u043e\\u0432\\u044b\\u0439 \\u043c\\u043e\\u043d\\u0438\\u0442\\u043e\\u0440\\u0438\\u043d\\u0433', u'23.188': u'\\u041d\\u0435\\u0434\\u0440\\u043e\\u043f\\u043e\\u043b\\u044c\\u0437\\u043e\\u0432\\u0430\\u043d\\u0438\\u0435', u'18.84': u'\\u0418\\u043d\\u0436\\u0435\\u043d\\u0435\\u0440, \\u041c\\u044f\\u0441\\u043e- \\u0438 \\u043f\\u0442\\u0438\\u0446\\u0435\\u043f\\u0435\\u0440\\u0435\\u0440\\u0430\\u0431\\u043e\\u0442\\u043a\\u0430', u'4.255': u'\\u0420\\u0435\\u0441\\u0435\\u043f\\u0448\\u0435\\u043d', u'20.490': u'\\u0420\\u0430\\u0431\\u043e\\u0447\\u0438\\u0435 \\u0441\\u0442\\u0440\\u043e\\u0438\\u0442\\u0435\\u043b\\u044c\\u043d\\u044b\\u0445 \\u0441\\u043f\\u0435\\u0446\\u0438\\u0430\\u043b\\u044c\\u043d\\u043e\\u0441\\u0442\\u0435\\u0439', u'23.182': u'\\u041d\\u0430\\u0447\\u0430\\u043b\\u044c\\u043d\\u044b\\u0439 \\u0443\\u0440\\u043e\\u0432\\u0435\\u043d\\u044c, \\u041c\\u0430\\u043b\\u043e \\u043e\\u043f\\u044b\\u0442\\u0430'}\n",
      "0.13013013013\n"
     ]
    }
   ],
   "source": [
    "###специализации\n",
    "import httplib\n",
    "import json\n",
    "headers = {\"Authorization\": \"Bearer PHUAM0L3PU56VNT041CJM3MUTGSABUCDTMBVHEAMF5CGCDEEIC7VFFT4VLOP0GQP\", \n",
    "           \"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"/specializations\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "\n",
    "#print json.loads(r1.read())\n",
    "specs = {}\n",
    "for k in [d['specializations'] for d in json.loads(r1.read())]:\n",
    "    for n in k:\n",
    "        specs[n['id']] = n['name']\n",
    "print specs\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100.3\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "###ЗП\n",
    "real = 100000.0\n",
    "expected = 100000.0\n",
    "print 1-(abs(expected-real)/max(real,expected))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n"
     ]
    }
   ],
   "source": [
    "##опыт работы\n",
    "real = 0\n",
    "expected = 0\n",
    "print (3.0 - abs(real-expected))/3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "##навыки\n",
    "import httplib\n",
    "import json\n",
    "import pickle\n",
    "headers = {\"Authorization\": \"Bearer UGE7369P4FJVPH027PQII1S422JHFJ2NQADKREFO8KTUP1BCIJQ2T6BUQ7CLQMGF\", \n",
    "           \"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"/specializations\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "\n",
    "# print json.loads(r1.read())\n",
    "spec_ids = []\n",
    "spec_names = []\n",
    "for k in [d['specializations'] for d in json.loads(r1.read())]:\n",
    "    for n in k:\n",
    "        spec_ids.append(n['id'])\n",
    "        spec_names.append(n['name'])\n",
    "pickle.dump( spec_ids, open( \"spec_ids.p\", \"wb\" ) )\n",
    "pickle.dump( spec_names, open( \"spec_names.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "199376\n",
      "6286\n",
      "1200\n"
     ]
    }
   ],
   "source": [
    "##вытаскиваем из вакансий ключевые навыки\n",
    "import json\n",
    "from tinydb import TinyDB\n",
    "import pickle\n",
    "import Stemmer\n",
    "import re \n",
    "\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "db = TinyDB('/home/shurik2533/vacancies.json')\n",
    "db2 = TinyDB('/home/shurik2533/vacancies2.json')\n",
    "vacancies = db.all()\n",
    "vacancies2 = db2.all()\n",
    "all_key_skills = []\n",
    "key_skills_set = set()\n",
    "key_skills_dict = dict()\n",
    "for vacancy in vacancies:\n",
    "#     print vacancy['name']\n",
    "    for skill in vacancy['key_skills']:\n",
    "        words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word)\n",
    "            all_key_skills.append(word)\n",
    "            key_skills_set.add(word)\n",
    "            if word in key_skills_dict:\n",
    "                key_skills_dict[word] = key_skills_dict[word]+1;\n",
    "            else:\n",
    "                key_skills_dict[word] = 1;\n",
    "#     print vacancy['description']\n",
    "\n",
    "for vacancy in vacancies2:\n",
    "#     print vacancy['name']\n",
    "    for skill in vacancy['key_skills']:\n",
    "        words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word)\n",
    "            all_key_skills.append(word)\n",
    "            key_skills_set.add(word)\n",
    "            if word in key_skills_dict:\n",
    "                key_skills_dict[word] = key_skills_dict[word]+1;\n",
    "            else:\n",
    "                key_skills_dict[word] = 1;\n",
    "#     print vacancy['description']\n",
    "\n",
    "\n",
    "print len(all_key_skills)\n",
    "print len(key_skills_set)\n",
    "for k in key_skills_dict.keys():\n",
    "    if key_skills_dict[k] <= 7:\n",
    "        del key_skills_dict[k]\n",
    "\n",
    "res = list(key_skills_dict.keys())\n",
    "print len(res)\n",
    "pickle.dump( res, open( \"key_skills.p\", \"wb\" ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1200\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "key_skills = pickle.load( open( \"key_skills.p\", \"rb\" ) )\n",
    "print len(key_skills)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44308\n",
      "11107\n",
      "1968\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from tinydb import TinyDB\n",
    "import pickle\n",
    "import Stemmer\n",
    "import re \n",
    "\n",
    "db = TinyDB('/home/shurik2533/vacancies.json')\n",
    "db2 = TinyDB('/home/shurik2533/vacancies2.json')\n",
    "vacancies1 = db.all()\n",
    "vacancies2 = db2.all()\n",
    "vacancies = vacancies1+vacancies2\n",
    "\n",
    "titles_s = set()\n",
    "for vac in vacancies:\n",
    "    titles_s.add(vac['name'])\n",
    "print len(titles_s)\n",
    "title_words = dict()\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "for title in titles_s:\n",
    "    title = re.sub(ur'[^a-zа-я]+', ' ', title.lower(), re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', title.strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word)\n",
    "        if word in title_words:\n",
    "            title_words[word] = title_words[word]+1;\n",
    "        else:\n",
    "            title_words[word] = 1;\n",
    "print len(title_words)\n",
    "for k in title_words.keys():\n",
    "    if title_words[k] <= 8 or len(k.strip()) <= 2:\n",
    "        del title_words[k]\n",
    "print len(title_words)\n",
    "res = list(title_words.keys())\n",
    "pickle.dump( res, open( \"title_words.p\", \"wb\" ) )\n",
    "# vacancies.append(vacancies2)\n",
    "# print len(vacancies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "113289\n",
      "290361\n",
      "137670\n",
      " юрист ведущ по претензион исков и договорн работ в област аренд недвижим в тр комплекс новомосковск в город московск ул хабаров м саларьев в администрац всег комплекс занима аренд коммерческ нежил помещен офис и торгов площади. обязанности: переговор с арендаторами. договорн работа. претензион - исков работ (иски, претенз и т. д.) представительств в арбитражн суд и административн процессе. требования: высш юридическ образование. хорош знан договорн работ в област аренд !!! оп работ с арбитражн суд в област «аренд недвижимости» ! условия: . гр/р: 5/2 , с 9.00 ч. – 18.00 ч. з/плата: пок 55 000 руб. - 60 000 руб. ( на рук ), в зависим от ваш опыта! карьерн рост : начальник юридическ отдела, с з/п до 100 000 руб. оплат телефон . оформлен по тк рф социальн пакет по тк рф в ближайш 3 - 4 месяц будет сво столов – бесплатн обеды! проезд: м. саларьево, 420 автобус, (10-12 минут), , остановк по требован «гостиничн комплекс.\n",
      "<p>Юрист (ведущий) по претензионно - исковой и договорной работе в области «Аренда недвижимости» в ТР Комплекс «НовоМосковский», в город Московский, ул. Хабарова (м. Саларьево). В администрацию всего Комплекса, занимающейся арендой коммерческих, нежилых помещений – офисы и торговые площади. </p> <p><strong>Обязанности: </strong></p> <ul> <li>Переговоры с Арендаторами.</li> <li>Договорная работа.</li> <li>Претензионно - исковая работа (иски, претензии и т. д.)</li> <li>Представительство в Арбитражном суде и административном процессе. </li> </ul> <p> <strong>Требования: </strong></p> <ul> <li>Высшее Юридическое образование. </li> <li>Хорошее знание Договорной работы в области Аренды !!!</li> <li>Опыт работы с Арбитражном судом в области «Аренда недвижимости» !</li> </ul> <p><strong>Условия: </strong>.</p> <ul> <li>Гр/р: 5/2 , с 9.00 ч. – 18.00 ч.</li> <li>З/плата: пока 55 000 руб. - 60 000 руб. ( на руки ), в зависимости от Вашего опыта!</li> <li>Карьерный рост : начальник юридического отдела, с з/п до 100 000 руб.</li> <li>Оплата телефона .</li> <li>Оформление по ТК РФ</li> <li>Социальный пакет по ТК РФ</li> <li>В ближайшие 3 - 4 месяца будет своя столовая – бесплатные обеды!</li> </ul> Проезд: М. Саларьево, 420 автобус, (10-12 минут), , остановка по требованию «Гостиничный Комплекс&quot;.\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from tinydb import TinyDB\n",
    "import pickle\n",
    "import Stemmer\n",
    "import re \n",
    "\n",
    "db = TinyDB('/home/shurik2533/vacancies.json')\n",
    "db2 = TinyDB('/home/shurik2533/vacancies2.json')\n",
    "vacancies1 = db.all()\n",
    "vacancies2 = db2.all()\n",
    "vacancies = vacancies1+vacancies2\n",
    "\n",
    "docs = []\n",
    "for vac in vacancies:\n",
    "    docs.append(vac['description'])\n",
    "print len(docs)\n",
    "    \n",
    "    \n",
    "doc_words = dict()\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "p_docs = []\n",
    "for doc in docs:\n",
    "    p_doc = \"\"\n",
    "    doc = re.sub('<[^>]*>', '', doc.lower())\n",
    "    doc = re.sub('&quot;', '', doc)\n",
    "    doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', doc.strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word)\n",
    "        p_doc = p_doc + \" \" + word\n",
    "        if word in doc_words:\n",
    "            doc_words[word] = doc_words[word]+1;\n",
    "        else:\n",
    "            doc_words[word] = 1;\n",
    "    p_docs.append(p_doc)\n",
    "print len(doc_words)\n",
    "for k in doc_words.keys():\n",
    "    if doc_words[k] == 1 or len(k.strip()) <= 1:\n",
    "        del doc_words[k]\n",
    "print len(doc_words)\n",
    "# pickle.dump( doc_words, open( \"doc_words.p\", \"wb\" ) )\n",
    "pickle.dump( p_docs, open( \"processed_docs.p\", \"wb\" ) )\n",
    "print p_docs[0]\n",
    "print docs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Юрист (ведущий) по претензионно - исковой и договорной работе в области «Аренда недвижимости» в ТР Комплекс «НовоМосковский», в город Московский, ул. Хабарова (м. Саларьево). В администрацию всего Комплекса, занимающейся арендой коммерческих, нежилых помещений – офисы и торговые площади.  Обязанности:   Переговоры с Арендаторами. Договорная работа. Претензионно - исковая работа (иски, претензии и т. д.) Представительство в Арбитражном суде и административном процессе.    Требования:   Высшее Юридическое образование.  Хорошее знание Договорной работы в области Аренды !!! Опыт работы с Арбитражном судом в области «Аренда недвижимости» !  Условия: .  Гр/р: 5/2 , с 9.00 ч. – 18.00 ч. З/плата: пока 55 000 руб. - 60 000 руб. ( на руки ), в зависимости от Вашего опыта! Карьерный рост : начальник юридического отдела, с з/п до 100 000 руб. Оплата телефона . Оформление по ТК РФ Социальный пакет по ТК РФ В ближайшие 3 - 4 месяца будет своя столовая – бесплатные обеды!  Проезд: М. Саларьево, 420 автобус, (10-12 минут), , остановка по требованию «Гостиничный Комплекс&quot;.\n"
     ]
    }
   ],
   "source": [
    "import re \n",
    "doc = \"<p>Юрист (ведущий) по претензионно - исковой и договорной работе в области «Аренда недвижимости» в ТР Комплекс «НовоМосковский», в город Московский, ул. Хабарова (м. Саларьево). В администрацию всего Комплекса, занимающейся арендой коммерческих, нежилых помещений – офисы и торговые площади. </p> <p><strong>Обязанности: </strong></p> <ul> <li>Переговоры с Арендаторами.</li> <li>Договорная работа.</li> <li>Претензионно - исковая работа (иски, претензии и т. д.)</li> <li>Представительство в Арбитражном суде и административном процессе. </li> </ul> <p> <strong>Требования: </strong></p> <ul> <li>Высшее Юридическое образование. </li> <li>Хорошее знание Договорной работы в области Аренды !!!</li> <li>Опыт работы с Арбитражном судом в области «Аренда недвижимости» !</li> </ul> <p><strong>Условия: </strong>.</p> <ul> <li>Гр/р: 5/2 , с 9.00 ч. – 18.00 ч.</li> <li>З/плата: пока 55 000 руб. - 60 000 руб. ( на руки ), в зависимости от Вашего опыта!</li> <li>Карьерный рост : начальник юридического отдела, с з/п до 100 000 руб.</li> <li>Оплата телефона .</li> <li>Оформление по ТК РФ</li> <li>Социальный пакет по ТК РФ</li> <li>В ближайшие 3 - 4 месяца будет своя столовая – бесплатные обеды!</li> </ul> Проезд: М. Саларьево, 420 автобус, (10-12 минут), , остановка по требованию «Гостиничный Комплекс&quot;.\"\n",
    "doc = re.sub('<[^>]*>', '', doc)\n",
    "print doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(113289, 120498)\n",
      "(113289, 3443)\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "\n",
    "doc_words = pickle.load( open( \"processed_docs.p\", \"rb\" ) )\n",
    "\n",
    "vectorizer = CountVectorizer(min_df=1)\n",
    "X = vectorizer.fit_transform(doc_words)\n",
    "# T = vectorizer.transform(['java разработчик программист'])\n",
    "\n",
    "print X.shape\n",
    "\n",
    "sel = VarianceThreshold(0.004)\n",
    "X2 = sel.fit_transform(X)\n",
    "# T2 = sel.transform(T)\n",
    "# print T2.toarray()\n",
    "\n",
    "print X2.shape\n",
    "\n",
    "transformer = TfidfTransformer()\n",
    "X2_tfidf = transformer.fit_transform(X2)\n",
    "\n",
    "pickle.dump( vectorizer, open( \"doc_vectorizer.p\", \"wb\" ) )\n",
    "pickle.dump( sel, open( \"count_vectorizer.p\", \"wb\" ) )\n",
    "pickle.dump( transformer, open( \"tfidf_transformer.p\", \"wb\" ) )\n",
    "\n",
    "# T2_tfidf = transformer.transform(T2)\n",
    "# result = cosine_similarity(T2_tfidf, X2_tfidf)\n",
    "\n",
    "# res = heapq.nlargest(10, range(len(result[0])), result[0].take)\n",
    "#print doc_words[result.argmax(axis=1)]\n",
    "# for i in res:\n",
    "#     print doc_words[i]\n",
    "   \n",
    "# from scipy import spatial\n",
    "# from sklearn.metrics.pairwise import cosine_similarity\n",
    "# i = 0\n",
    "# for x_row in X2_tfidf:\n",
    "#     result = cosine_similarity(T2_tfidf, x_row)\n",
    "#     #result = spatial.distance.cosine(, )\n",
    "#     print result\n",
    "#     i = i+1\n",
    "#     if i > 10:\n",
    "#         break\n",
    "#result = spatial.distance.cosine(T2_tfidf, X2_tfidf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/vacancies?per_page=200&date_from=2016-06-22T13:06:04&date_to=2016-06-22T13:26:04&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:06:04&date_to=2016-06-22T13:26:04&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:06:04&date_to=2016-06-22T13:26:04&page=2\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:06:04&date_to=2016-06-22T13:26:04&page=3\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:06:04&date_to=2016-06-22T13:26:04&page=4\n",
      "802\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "starting t6\n",
      "starting t7\n",
      "starting t8\n",
      "starting t9\n",
      "60 sec\n"
     ]
    }
   ],
   "source": [
    "import httplib\n",
    "import json\n",
    "import datetime\n",
    "import MySQLdb\n",
    "import ConfigParser\n",
    "import time\n",
    "import threading\n",
    "from dateutil.parser import parse\n",
    "\n",
    "current_time = lambda: int(round(time.time()))\n",
    "start = current_time()\n",
    "    \n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "\n",
    "config = ConfigParser.ConfigParser()\n",
    "config.readfp(open('my.cfg'))\n",
    "\n",
    "db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                     port=config.getint('mysqld', 'port'), \n",
    "                     user=config.get('mysqld', 'user'), \n",
    "                     passwd=config.get('mysqld', 'password'), \n",
    "                     db=config.get('mysqld', 'database') )\n",
    "db.set_character_set('utf8')\n",
    "cursor = db.cursor()\n",
    "cursor.execute('SET NAMES utf8;')\n",
    "cursor.execute('SET CHARACTER SET utf8;')\n",
    "cursor.execute('SET character_set_connection=utf8;')\n",
    "cursor.close()\n",
    "\n",
    "def get_vacancy_ids():\n",
    "    vacancy_ids = []\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    per_page = 200\n",
    "    page = 0\n",
    "    count = per_page\n",
    "    date_from = (datetime.datetime.now() - datetime.timedelta(minutes=5)).strftime('%Y-%m-%dT%H:%M:%S')\n",
    "    date_to = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')\n",
    "    while count == per_page:\n",
    "        path = (\"/vacancies?per_page={}&date_from={}&date_to={}&page={}\"\n",
    "                .format(per_page, date_from, date_to, page))\n",
    "        print path\n",
    "\n",
    "        conn.request(\"GET\", path, headers=headers)\n",
    "        r1 = conn.getresponse()\n",
    "        vacancies = r1.read()\n",
    "        try:\n",
    "            count = len(json.loads(vacancies)['items'])\n",
    "        except KeyError:\n",
    "            print vacancies\n",
    "        page = page+1\n",
    "        for item in json.loads(vacancies)['items']:\n",
    "            vacancy_ids.append(item['id'])\n",
    "    return vacancy_ids\n",
    "        \n",
    "\n",
    "def process_vacancies(vacancy_ids):\n",
    "    headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "\n",
    "    config = ConfigParser.ConfigParser()\n",
    "    config.readfp(open('my.cfg'))\n",
    "\n",
    "    db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                         port=config.getint('mysqld', 'port'), \n",
    "                         user=config.get('mysqld', 'user'), \n",
    "                         passwd=config.get('mysqld', 'password'), \n",
    "                         db=config.get('mysqld', 'database') )\n",
    "    db.set_character_set('utf8')\n",
    "    cursor = db.cursor()\n",
    "    cursor.execute('SET NAMES utf8;')\n",
    "    cursor.execute('SET CHARACTER SET utf8;')\n",
    "    cursor.execute('SET character_set_connection=utf8;')\n",
    "    cursor.close()\n",
    "\n",
    "    for vac_id in vacancy_ids:\n",
    "        conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "        conn.request(\"GET\", \"/vacancies/{}\".format(vac_id), headers=headers)\n",
    "        r1 = conn.getresponse()\n",
    "        vacancy = r1.read()\n",
    "        vacancy_json = json.loads(vacancy)\n",
    "        cursor = db.cursor()\n",
    "        cursor.execute(\"\"\"\n",
    "          INSERT INTO vacancies (id, updated, item) \n",
    "          VALUES (%s, %s, %s)\n",
    "          ON DUPLICATE KEY UPDATE \n",
    "            updated  = VALUES(updated), \n",
    "            item   = VALUES(item)\"\"\", (vac_id, \n",
    "                                       parse(vacancy_json['published_at']).strftime(\"%Y-%m-%d %H:%M:%S\"), \n",
    "                                       vacancy))\n",
    "        db.commit() \n",
    "        cursor.close()\n",
    "    db.close()\n",
    "\n",
    "ids = get_vacancy_ids()\n",
    "print len(ids)\n",
    "vac_id_chunks=[ids[x:x+100] for x in xrange(0, len(ids), 100)]\n",
    "t_num = 1;\n",
    "threads = []\n",
    "for vac_id_chunk in vac_id_chunks:\n",
    "    print 'starting t{}'.format(t_num)\n",
    "    t_num = t_num + 1\n",
    "    t = threading.Thread(target=process_vacancies, kwargs={'vacancy_ids': vac_id_chunk})\n",
    "    threads.append(t)\n",
    "    t.start()\n",
    "    \n",
    "for t in threads:\n",
    "    t.join()\n",
    "    \n",
    "\n",
    "db.close()\n",
    "print \"{} sec\".format(current_time()-start)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done\n"
     ]
    }
   ],
   "source": [
    "#vacancy vectorizer\n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "from tinydb import TinyDB\n",
    "import ConfigParser\n",
    "import MySQLdb\n",
    "import json\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "import httplib\n",
    "import re \n",
    "import Stemmer\n",
    "\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "config = ConfigParser.ConfigParser()\n",
    "config.readfp(open('my.cfg'))\n",
    "db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                     port=config.getint('mysqld', 'port'), \n",
    "                     user=config.get('mysqld', 'user'), \n",
    "                     passwd=config.get('mysqld', 'password'), \n",
    "                     db=config.get('mysqld', 'database') )\n",
    "db.set_character_set('utf8')\n",
    "cursor = db.cursor()\n",
    "cursor.execute('SET NAMES utf8;')\n",
    "cursor.execute('SET CHARACTER SET utf8;')\n",
    "cursor.execute('SET character_set_connection=utf8;')\n",
    "cursor = db.cursor()\n",
    "cursor.execute(\"\"\"SELECT item FROM vacancies WHERE updated >= (NOW() - INTERVAL 7 DAY) LIMIT 10\"\"\")\n",
    "\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "features_by_type = {}\n",
    "for employment_type in dictionaries_json['employment']:\n",
    "    features_by_type[employment_type['id']] = []\n",
    "\n",
    "spec_ids = pickle.load( open( \"spec_ids.p\", \"rb\" ) )\n",
    "key_skills = pickle.load( open( \"key_skills.p\", \"rb\" ) )\n",
    "title_words = pickle.load( open( \"title_words.p\", \"rb\" ) )\n",
    "\n",
    "vectorizer = pickle.load( open( \"doc_vectorizer.p\", \"rb\" ) )\n",
    "count_vectorizer = pickle.load( open( \"count_vectorizer.p\", \"rb\" ) )\n",
    "tfidf_transformer = pickle.load( open( \"tfidf_transformer.p\", \"rb\" ) )\n",
    "\n",
    "# # специацизации | уровни зп| навыки (собираем из вакансий) | \n",
    "# # список слов заголовка (собираем из вакансий) | текст (собираем из вакансий)\n",
    "for item in cursor:\n",
    "    feature = []\n",
    "    vacancy = json.loads(item[0])\n",
    "    \n",
    "    #specializations\n",
    "    vac_specializations = vacancy['specializations']\n",
    "    spec_features = [0] * len(spec_ids)\n",
    "    for vac_spec in vac_specializations:\n",
    "        spec_features[spec_ids.index(vac_spec['id'])] = 1        \n",
    "    feature = feature + spec_features\n",
    "    \n",
    "    #salary\n",
    "    salary = 0\n",
    "    if vacancy['salary'] != None:\n",
    "        if vacancy['salary']['from'] == None and vacancy['salary']['to'] != None:\n",
    "            salary = vacancy['salary']['to']/currency_rates[vacancy['salary']['currency']]\n",
    "        elif vacancy['salary']['to'] == None and vacancy['salary']['from'] != None:\n",
    "            salary = vacancy['salary']['from']/currency_rates[vacancy['salary']['currency']]\n",
    "        elif vacancy['salary']['to'] != None and vacancy['salary']['from'] != None:\n",
    "            salary = ((vacancy['salary']['from'] + vacancy['salary']['to'])/2)/currency_rates[vacancy['salary']['currency']]\n",
    "    max_salary = 500000.0\n",
    "    if salary >= max_salary:\n",
    "        salary = max_salary\n",
    "    salary = salary/max_salary\n",
    "    feature.append(salary)\n",
    "    \n",
    "    #keyskills\n",
    "    skill_features = [0] * len(key_skills)\n",
    "    vac_skills = vacancy['key_skills']\n",
    "    for skill in vac_skills:\n",
    "        words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word)\n",
    "            if word in key_skills:\n",
    "                skill_features[key_skills.index(word)] = 1 \n",
    "    feature = feature + skill_features\n",
    "    \n",
    "    #title\n",
    "    title_features = [0] * len(title_words)\n",
    "    title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', title.strip())\n",
    "    for title_word in words:\n",
    "        title_word = stemmer.stemWord(title_word)\n",
    "        if title_word in title_words:\n",
    "            title_features[title_words.index(title_word)] = 1\n",
    "    feature = feature + title_features\n",
    "    \n",
    "    #description\n",
    "    p_doc = \"\"\n",
    "    doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "    doc = re.sub('&quot;', '', doc)\n",
    "    doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', doc.strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word)\n",
    "        p_doc = p_doc + \" \" + word\n",
    "    p_doc_vec = vectorizer.transform([p_doc])\n",
    "    p_doc_vec2 = count_vectorizer.transform(p_doc_vec)\n",
    "    p_doc_vec_tfidf = tfidf_transformer.transform(p_doc_vec2)\n",
    "    feature = feature + list(p_doc_vec_tfidf.toarray()[0])\n",
    "    features_by_type[vacancy['employment']['id']].append(feature)\n",
    "\n",
    "pickle.dump( features_by_type, open( \"features_by_type.p\", \"wb\" ) )\n",
    "cursor.close()\n",
    "\n",
    "db.close()\n",
    "\n",
    "print('done')\n",
    "    \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27cf52c1ff00b6dc6d0039ed1f736563726574\n",
      "Java Developer\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "lower not found",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-12-ea316ec2e81d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    120\u001b[0m     \u001b[0mp_doc\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mp_doc\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\" \"\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mword\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    121\u001b[0m \u001b[0mp_doc_vec\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mvectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mp_doc\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 122\u001b[1;33m \u001b[0mp_doc_vec2\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcount_vectorizer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp_doc_vec\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    123\u001b[0m \u001b[0mp_doc_vec_tfidf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtfidf_transformer\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp_doc_vec2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    124\u001b[0m \u001b[0mfeature\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfeature\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mp_doc_vec_tfidf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtoarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/shurik2533/anaconda/lib/python2.7/site-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36mtransform\u001b[1;34m(self, raw_documents)\u001b[0m\n\u001b[0;32m    864\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    865\u001b[0m         \u001b[1;31m# use the same matrix-building strategy as fit_transform\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 866\u001b[1;33m         \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_count_vocab\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mraw_documents\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfixed_vocab\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    867\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbinary\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    868\u001b[0m             \u001b[0mX\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfill\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/shurik2533/anaconda/lib/python2.7/site-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m_count_vocab\u001b[1;34m(self, raw_documents, fixed_vocab)\u001b[0m\n\u001b[0;32m    750\u001b[0m         \u001b[0mindptr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    751\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mdoc\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mraw_documents\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 752\u001b[1;33m             \u001b[1;32mfor\u001b[0m \u001b[0mfeature\u001b[0m \u001b[1;32min\u001b[0m \u001b[0manalyze\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    753\u001b[0m                 \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    754\u001b[0m                     \u001b[0mj_indices\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvocabulary\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mfeature\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/shurik2533/anaconda/lib/python2.7/site-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(doc)\u001b[0m\n\u001b[0;32m    236\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    237\u001b[0m             return lambda doc: self._word_ngrams(\n\u001b[1;32m--> 238\u001b[1;33m                 tokenize(preprocess(self.decode(doc))), stop_words)\n\u001b[0m\u001b[0;32m    239\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    240\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/shurik2533/anaconda/lib/python2.7/site-packages/sklearn/feature_extraction/text.pyc\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    202\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    203\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlowercase\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 204\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[1;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mstrip_accents\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mlower\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    205\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    206\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mstrip_accents\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/shurik2533/anaconda/lib/python2.7/site-packages/scipy/sparse/base.pyc\u001b[0m in \u001b[0;36m__getattr__\u001b[1;34m(self, attr)\u001b[0m\n\u001b[0;32m    506\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mgetnnz\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    507\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 508\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mattr\u001b[0m \u001b[1;33m+\u001b[0m \u001b[1;34m\" not found\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    509\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    510\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mtranspose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: lower not found"
     ]
    }
   ],
   "source": [
    "#resume vectorizer\n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "from tinydb import TinyDB\n",
    "import ConfigParser\n",
    "import MySQLdb\n",
    "import json\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "import httplib\n",
    "import re \n",
    "import Stemmer\n",
    "\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "headers = {\"User-Agent\": \"hh-recommender\", \"Authorization\" : \"Bearer T5MIT6GVV85LSVR75CB7U768TR3PFGS990I3QJNFV6A4CBQJF6M30G0MOT8U2V8I\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/resumes/mine\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "me = r1.read()\n",
    "me_json = json.loads(me)\n",
    "print me_json['items'][0]['id']\n",
    "\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/resumes/{}\".format(me_json['items'][0]['id']), headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "resume = r1.read()\n",
    "resume_json = json.loads(resume)\n",
    "\n",
    "feature = []\n",
    "resume_type = 'full'\n",
    "\n",
    "spec_ids = pickle.load( open( \"spec_ids.p\", \"rb\" ) )\n",
    "key_skills = pickle.load( open( \"key_skills.p\", \"rb\" ) )\n",
    "title_words = pickle.load( open( \"title_words.p\", \"rb\" ) )\n",
    "\n",
    "vectorizer = pickle.load( open( \"doc_vectorizer.p\", \"rb\" ) )\n",
    "count_vectorizer = pickle.load( open( \"count_vectorizer.p\", \"rb\" ) )\n",
    "tfidf_transformer = pickle.load( open( \"tfidf_transformer.p\", \"rb\" ) )\n",
    "\n",
    "# # специацизации | уровни зп| навыки (собираем из вакансий) | \n",
    "# # список слов заголовка (собираем из вакансий) | текст (собираем из вакансий)\n",
    "\n",
    "resume_type = resume_json['employment']['id']\n",
    "\n",
    "#specializations\n",
    "res_specializations = resume_json['specialization']\n",
    "spec_features = [0] * len(spec_ids)\n",
    "for res_spec in res_specializations:\n",
    "    spec_features[spec_ids.index(res_spec['id'])] = 1        \n",
    "feature = feature + spec_features\n",
    "\n",
    "#salary\n",
    "salary = 0\n",
    "if resume_json['salary'] != None and resume_json['salary']['amount'] != None:\n",
    "    salary = resume_json['salary']['amount']/currency_rates[resume_json['salary']['currency']]\n",
    "max_salary = 500000.0\n",
    "if salary >= max_salary:\n",
    "    salary = max_salary\n",
    "salary = salary/max_salary\n",
    "feature.append(salary)\n",
    "\n",
    "#keyskills\n",
    "skill_features = [0] * len(key_skills)\n",
    "res_skills = resume_json['skill_set']\n",
    "for skill in res_skills:\n",
    "    words = re.split(r'\\s{1,}', skill.lower().strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word)\n",
    "        if word in key_skills:\n",
    "            skill_features[key_skills.index(word)] = 1 \n",
    "feature = feature + skill_features\n",
    "\n",
    "#title\n",
    "title_features = [0] * len(title_words)\n",
    "title = re.sub(ur'[^a-zа-я]+', ' ', resume_json['title'].lower(), re.UNICODE)\n",
    "words = re.split(r'\\s{1,}', title.strip())\n",
    "for title_word in words:\n",
    "    title_word = stemmer.stemWord(title_word)\n",
    "    if title_word in title_words:\n",
    "        title_features[title_words.index(title_word)] = 1\n",
    "feature = feature + title_features\n",
    "\n",
    "#description\n",
    "doc = resume_json['skills']\n",
    "if resume_json['experience'] != None and len(resume_json['experience']) > 0:\n",
    "    doc =  doc + \" \" + resume_json['experience'][0]['description']\n",
    "p_doc = \"\"\n",
    "doc = re.sub('<[^>]*>', '', doc.lower())\n",
    "doc = re.sub('&quot;', '', doc)\n",
    "doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "words = re.split(r'\\s{1,}', doc.strip())\n",
    "for word in words:\n",
    "    word = stemmer.stemWord(word)\n",
    "    p_doc = p_doc + \" \" + word\n",
    "p_doc_vec = vectorizer.transform([p_doc])\n",
    "p_doc_vec2 = count_vectorizer.transform(p_doc_vec)\n",
    "p_doc_vec_tfidf = tfidf_transformer.transform(p_doc_vec2)\n",
    "feature = feature + list(p_doc_vec_tfidf.toarray()[0])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vacancies done\n",
      "27cf52c1ff00b6dc6d0039ed1f736563726574\n",
      " опытн разработчик программн обеспечен им широк спектр навык знан технолог разрабатыва быстр программ работа больш объем дан многопотоков сред уделя больш вниман качеств код удобств дальн поддержк аналитическ подхож решен задач. им хорош знан алгоритм структур данных. хобби, увлечен интернет-технологии, путешествия, фотография, кулинария. организова вед местн школ кружок по программирован для детей. последн врем интерес машин обучен opencv профил на habrahabr: http://habrahabr.ru/users/shurik2533/  java developer  java java ee oracle pl/sql python data analysis big data recommender systems machine learning\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "docs processed\n",
      "9924\n",
      "finish\n"
     ]
    }
   ],
   "source": [
    "#из каждой профобласти извлекаем ключевые слова, релевантные для нее.\n",
    "#есть и общие слова. каждая профобласть должна иметь свой вектор множителей весов\n",
    "#верная зп должна добавлять какой-то вес (сортировать результат)\n",
    "#для каждой профобласти считать свой tfidf? и вектор кол-ва найденных слов умножать на него?\n",
    "#не забыть учитывать тип занятости\n",
    "\n",
    "#собираем вектор из таких фич, чтобы определенная часть в большей степени соответствовала определенной специялизации\n",
    "#тогда потом, когда будем считать косинусное расстояние будет больше созпадать с нужной профобластью\n",
    "\n",
    "\n",
    "#!считал не верно. считаем tfidf для каждой специализации. на основе этого выбираем самые значимые из специализации\n",
    "#слова из которых затем строитятся вектора. Попробовать использовать vocabulary\n",
    "import json\n",
    "from tinydb import TinyDB\n",
    "import pickle\n",
    "import Stemmer\n",
    "import re \n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "\n",
    "db = TinyDB('vacancies.json')\n",
    "db2 = TinyDB('vacancies2.json')\n",
    "vacancies1 = db.all()\n",
    "vacancies2 = db2.all()\n",
    "vacancies = vacancies1+vacancies2\n",
    "\n",
    "# db = TinyDB('vacancies.json')\n",
    "# vacancies = db.all()\n",
    "\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "\n",
    "vac_by_spec = {}\n",
    "i = 0\n",
    "#keyskills, title, description\n",
    "for vacancy in vacancies:\n",
    "    #description\n",
    "    p_doc = ''\n",
    "    doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "    doc = re.sub('&quot;', '', doc)\n",
    "    doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', doc.strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word.strip())\n",
    "        if len(word.strip()) > 1:\n",
    "            p_doc = p_doc + \" \" + word\n",
    "            \n",
    "    #title\n",
    "    p_title = ''\n",
    "    title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', title.strip())\n",
    "    for title_word in words:\n",
    "        title_word = stemmer.stemWord(title_word)\n",
    "        if len(title_word.strip()) > 1:\n",
    "            p_title = p_title + \" \" + title_word.strip()\n",
    "    \n",
    "    #keyskills\n",
    "    p_skills = ''\n",
    "    vac_skills = vacancy['key_skills']\n",
    "    for skill in vac_skills:\n",
    "        words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word)\n",
    "            if len(word.strip()) > 1:\n",
    "                p_skills = p_skills + \" \" + word.strip()\n",
    "    \n",
    "    p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "    \n",
    "    for spec in vacancy['specializations']:\n",
    "        if spec['id'] not in vac_by_spec:\n",
    "            vac_by_spec[spec['id']] = []\n",
    "        vac_by_spec[spec['id']].append(p_doc)\n",
    "    \n",
    "#     i = i+1\n",
    "#     if i > 100:\n",
    "#         break\n",
    "print 'docs processed'\n",
    "words = set()\n",
    "for key in vac_by_spec:\n",
    "    corpus = vac_by_spec[key]\n",
    "    vectorizer = CountVectorizer(min_df=1)\n",
    "    X = vectorizer.fit_transform(corpus)\n",
    "    transformer = TfidfTransformer()\n",
    "    X_tfidf = transformer.fit_transform(X)\n",
    "    spec_means = X_tfidf.mean(axis=0)\n",
    "    spec_means_arr = numpy.squeeze(numpy.asarray(spec_means))\n",
    "    res = heapq.nlargest(350, range(len(spec_means_arr)), spec_means_arr.take)\n",
    "    for i in res:\n",
    "        words.add(vectorizer.get_feature_names()[i])\n",
    "        \n",
    "print len(words)\n",
    "pickle.dump( words, open( \"spec_dict2.p\", \"wb\" ) )\n",
    "print 'finish'\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.6316672   0.44943642  0.6316672   0.        ]\n",
      " [ 0.          0.81818021  0.          0.57496187]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "corpus = ['aa bb cc', 'bb bb dd']\n",
    "vectorizer = CountVectorizer(min_df=1)\n",
    "X = vectorizer.fit_transform(corpus)\n",
    "transformer = TfidfTransformer()\n",
    "X_tfidf = transformer.fit_transform(X)\n",
    "print X_tfidf.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish collect docs\n",
      "finish\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from tinydb import TinyDB\n",
    "import pickle\n",
    "import Stemmer\n",
    "import re \n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "\n",
    "db = TinyDB('vacancies.json')\n",
    "db2 = TinyDB('vacancies2.json')\n",
    "vacancies1 = db.all()\n",
    "vacancies2 = db2.all()\n",
    "vacancies = vacancies1+vacancies2\n",
    "\n",
    "# db = TinyDB('vacancies.json')\n",
    "# vacancies = db.all()\n",
    "\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "\n",
    "i = 0\n",
    "p_docs = []\n",
    "#keyskills, title, description\n",
    "for vacancy in vacancies:\n",
    "    #description\n",
    "    p_doc = ''\n",
    "    doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "    doc = re.sub('&quot;', '', doc)\n",
    "    doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', doc.strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word.strip())\n",
    "        if len(word.strip()) > 1:\n",
    "            p_doc = p_doc + \" \" + word\n",
    "            \n",
    "    #title\n",
    "    p_title = ''\n",
    "    title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', title.strip())\n",
    "    for title_word in words:\n",
    "        title_word = stemmer.stemWord(title_word)\n",
    "        if len(title_word.strip()) > 1:\n",
    "            p_title = p_title + \" \" + title_word.strip()\n",
    "    \n",
    "    #keyskills\n",
    "    p_skills = ''\n",
    "    vac_skills = vacancy['key_skills']\n",
    "    for skill in vac_skills:\n",
    "        words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word)\n",
    "            if len(word.strip()) > 1:\n",
    "                p_skills = p_skills + \" \" + word.strip()\n",
    "    \n",
    "    p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "    p_docs.append(p_doc)\n",
    "    \n",
    "print 'finish collect docs'\n",
    "\n",
    "voc = pickle.load( open( \"spec_dict2.p\", \"rb\" ) )\n",
    "vectorizer = CountVectorizer(min_df=1, vocabulary=voc)\n",
    "X = vectorizer.fit_transform(p_docs)\n",
    "transformer = TfidfTransformer()\n",
    "X_tfidf = transformer.fit_transform(X)\n",
    "pickle.dump( vectorizer, open( \"count_vectorizer.p\", \"wb\" ) )\n",
    "pickle.dump( transformer, open( \"tfidf_transformer.p\", \"wb\" ) )\n",
    "\n",
    "print 'finish'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish 113289.0 - 35969.0 - 0.317497727052\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from tinydb import TinyDB\n",
    "import pickle\n",
    "import Stemmer\n",
    "import re \n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "\n",
    "db = TinyDB('vacancies.json')\n",
    "db2 = TinyDB('vacancies2.json')\n",
    "vacancies1 = db.all()\n",
    "vacancies2 = db2.all()\n",
    "vacancies = vacancies1+vacancies2\n",
    "\n",
    "# db = TinyDB('vacancies.json')\n",
    "# vacancies = db.all()\n",
    "\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "\n",
    "i = 0.0\n",
    "p_docs = []\n",
    "none = 0.0\n",
    "#keyskills, title, description\n",
    "for vacancy in vacancies:\n",
    "    if vacancy['salary'] == None:\n",
    "        none = none+1\n",
    "    i = i+1\n",
    "#     if i > 10:\n",
    "#         break\n",
    "    \n",
    "print 'finish {} - {} - {}'.format(i, none, none/i)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27cf52c1ff00b6dc6d0039ed1f736563726574\n",
      "vacancies done\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/shurik2533/anaconda/lib/python2.7/site-packages/sklearn/utils/validation.py:386: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and willraise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Found array with 0 feature(s) (shape=(1, 0)) while a minimum of 1 is required.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-3c680237e6be>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m    186\u001b[0m \u001b[0mfeatures\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfeatures_by_type\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mresume_type\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    187\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 188\u001b[1;33m \u001b[0mresult\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcosine_similarity\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfeature\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeatures\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    189\u001b[0m \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mheapq\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mnlargest\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m10\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mrange\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    190\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/shurik2533/anaconda/lib/python2.7/site-packages/sklearn/metrics/pairwise.pyc\u001b[0m in \u001b[0;36mcosine_similarity\u001b[1;34m(X, Y, dense_output)\u001b[0m\n\u001b[0;32m    879\u001b[0m     \u001b[1;31m# to avoid recursive import\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    880\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 881\u001b[1;33m     \u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_pairwise_arrays\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    882\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    883\u001b[0m     \u001b[0mX_normalized\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnormalize\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/shurik2533/anaconda/lib/python2.7/site-packages/sklearn/metrics/pairwise.pyc\u001b[0m in \u001b[0;36mcheck_pairwise_arrays\u001b[1;34m(X, Y, precomputed)\u001b[0m\n\u001b[0;32m     94\u001b[0m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mY\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'csr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     95\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 96\u001b[1;33m         \u001b[0mX\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'csr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     97\u001b[0m         \u001b[0mY\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcheck_array\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mY\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maccept_sparse\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'csr'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdtype\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     98\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/home/shurik2533/anaconda/lib/python2.7/site-packages/sklearn/utils/validation.pyc\u001b[0m in \u001b[0;36mcheck_array\u001b[1;34m(array, accept_sparse, dtype, order, copy, force_all_finite, ensure_2d, allow_nd, ensure_min_samples, ensure_min_features, warn_on_dtype, estimator)\u001b[0m\n\u001b[0;32m    413\u001b[0m                              \u001b[1;34m\" a minimum of %d is required%s.\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    414\u001b[0m                              % (n_features, shape_repr, ensure_min_features,\n\u001b[1;32m--> 415\u001b[1;33m                                 context))\n\u001b[0m\u001b[0;32m    416\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    417\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mwarn_on_dtype\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mdtype_orig\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0marray\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mdtype_orig\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Found array with 0 feature(s) (shape=(1, 0)) while a minimum of 1 is required."
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "from tinydb import TinyDB\n",
    "import ConfigParser\n",
    "import MySQLdb\n",
    "import json\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "import httplib\n",
    "import re \n",
    "import Stemmer\n",
    "\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "config = ConfigParser.ConfigParser()\n",
    "config.readfp(open('my.cfg'))\n",
    "db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                     port=config.getint('mysqld', 'port'), \n",
    "                     user=config.get('mysqld', 'user'), \n",
    "                     passwd=config.get('mysqld', 'password'), \n",
    "                     db=config.get('mysqld', 'database') )\n",
    "db.set_character_set('utf8')\n",
    "cursor = db.cursor()\n",
    "cursor.execute('SET NAMES utf8;')\n",
    "cursor.execute('SET CHARACTER SET utf8;')\n",
    "cursor.execute('SET character_set_connection=utf8;')\n",
    "\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "features_by_type = {}\n",
    "for employment_type in dictionaries_json['employment']:\n",
    "    features_by_type[employment_type['id']] = []\n",
    "\n",
    "spec_ids = pickle.load( open( \"spec_ids.p\", \"rb\" ) )\n",
    "key_skills = pickle.load( open( \"key_skills.p\", \"rb\" ) )\n",
    "title_words = pickle.load( open( \"title_words.p\", \"rb\" ) )\n",
    "\n",
    "count_vectorizer = pickle.load( open( \"count_vectorizer.p\", \"rb\" ) )\n",
    "tfidf_transformer = pickle.load( open( \"tfidf_transformer.p\", \"rb\" ) )\n",
    "\n",
    "\n",
    "cursor = db.cursor()\n",
    "cursor.execute(\"\"\"SELECT item, id FROM vacancies WHERE updated >= (NOW() - INTERVAL 7 DAY) LIMIT 10\"\"\")\n",
    "data = []\n",
    "for item in cursor:\n",
    "    feature = []\n",
    "    vacancy = json.loads(item[0])\n",
    "    data.append(vacancy['name'])\n",
    "    \n",
    "    #description\n",
    "    p_doc = ''\n",
    "    doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "    doc = re.sub('&quot;', '', doc)\n",
    "    doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', doc.strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word.strip())\n",
    "        if len(word.strip()) > 1:\n",
    "            p_doc = p_doc + \" \" + word\n",
    "            \n",
    "    #title\n",
    "    p_title = ''\n",
    "    title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', title.strip())\n",
    "    for title_word in words:\n",
    "        title_word = stemmer.stemWord(title_word)\n",
    "        if len(title_word.strip()) > 1:\n",
    "            p_title = p_title + \" \" + title_word.strip()\n",
    "    \n",
    "    #keyskills\n",
    "    p_skills = ''\n",
    "    vac_skills = vacancy['key_skills']\n",
    "    for skill in vac_skills:\n",
    "        words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word)\n",
    "            if len(word.strip()) > 1:\n",
    "                p_skills = p_skills + \" \" + word.strip()\n",
    "    \n",
    "    p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "    \n",
    "    feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "    tfidf_feature_p_doc = tfidf_transformer.transform(feature_p_doc)\n",
    "    \n",
    "    if features_by_type[vacancy['employment']['id']] == None:\n",
    "        features_by_type[vacancy['employment']['id']] = []\n",
    "    features_by_type[vacancy['employment']['id']].append(tfidf_feature_p_doc.toarray()[0])\n",
    "\n",
    "cursor.close()\n",
    "\n",
    "db.close()\n",
    "\n",
    "print('vacancies done')\n",
    "\n",
    "\n",
    "\n",
    "headers = {\"User-Agent\": \"hh-recommender\", \"Authorization\" : \"Bearer T5MIT6GVV85LSVR75CB7U768TR3PFGS990I3QJNFV6A4CBQJF6M30G0MOT8U2V8I\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/resumes/mine\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "me = r1.read()\n",
    "me_json = json.loads(me)\n",
    "\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/resumes/{}\".format(me_json['items'][0]['id']), headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "resume = r1.read()\n",
    "resume_json = json.loads(resume)\n",
    "\n",
    "feature = []\n",
    "resume_type = 'full'\n",
    "\n",
    "resume_type = resume_json['employment']['id']\n",
    "\n",
    "#description\n",
    "p_doc = ''\n",
    "doc = re.sub('<[^>]*>', '', resume_json['skills'].lower())\n",
    "doc = re.sub('&quot;', '', doc)\n",
    "doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "words = re.split(r'\\s{1,}', doc.strip())\n",
    "for word in words:\n",
    "    word = stemmer.stemWord(word.strip())\n",
    "    if len(word.strip()) > 1:\n",
    "        p_doc = p_doc + \" \" + word\n",
    "\n",
    "#title\n",
    "p_title = ''\n",
    "title = re.sub(ur'[^a-zа-я]+', ' ', resume_json['title'].lower(), re.UNICODE)\n",
    "words = re.split(r'\\s{1,}', title.strip())\n",
    "for title_word in words:\n",
    "    title_word = stemmer.stemWord(title_word)\n",
    "    if len(title_word.strip()) > 1:\n",
    "        p_title = p_title + \" \" + title_word.strip()\n",
    "\n",
    "#keyskills\n",
    "p_skills = ''\n",
    "res_skills = resume_json['skill_set']\n",
    "for skill in res_skills:\n",
    "    words = re.split(r'\\s{1,}', skill.lower().strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word)\n",
    "        if len(word.strip()) > 1:\n",
    "            p_skills = p_skills + \" \" + word.strip()\n",
    "\n",
    "p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "\n",
    "feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "feature = tfidf_transformer.transform(feature_p_doc)\n",
    "\n",
    "features = features_by_type[resume_type]\n",
    "\n",
    "result = cosine_similarity(feature, features)\n",
    "res = heapq.nlargest(10, range(len(result[0])), result[0].take)\n",
    "for i in res:\n",
    "    print result[0][i]\n",
    "    print data[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9924\n",
      "1\n",
      "1\n",
      "1\n",
      "1\n",
      "9924\n",
      "0.583040144288\n",
      "0.518259144701\n",
      "0.448771156518\n",
      "0.435977176192\n"
     ]
    }
   ],
   "source": [
    "count_vectorizer = pickle.load( open( \"count_vectorizer.p\", \"rb\" ) )\n",
    "tfidf_transformer = pickle.load( open( \"tfidf_transformer.p\", \"rb\" ) )\n",
    "doc = \"java разработчик программист api rrreree\"\n",
    "X = count_vectorizer.transform([doc])\n",
    "X1 = tfidf_transformer.transform(X)\n",
    "print len(X.toarray()[0])\n",
    "for v in X.toarray()[0]:\n",
    "    if v != 0:\n",
    "        print v\n",
    "\n",
    "print len(X1.toarray()[0])\n",
    "for v in X1.toarray()[0]:\n",
    "    if v != 0:\n",
    "        print v\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " любл готов вредн привычек не им аккуратн  повар  поварск дел\n",
      "processed 1000 vаcancies\n",
      "processed 2000 vаcancies\n",
      "processed 3000 vаcancies\n",
      "processed 4000 vаcancies\n",
      "processed 5000 vаcancies\n",
      "processed 6000 vаcancies\n",
      "processed 7000 vаcancies\n",
      "processed 8000 vаcancies\n",
      "processed 9000 vаcancies\n",
      "processed 10000 vаcancies\n",
      "0.369745464628\n",
      "17136190\n",
      "0.352538100623\n",
      "16764555\n",
      "0.28401864007\n",
      "17135847\n",
      "0.28179223153\n",
      "17135823\n",
      "0.2696830206\n",
      "17137164\n",
      "0.265194214145\n",
      "17134022\n",
      "0.239828494588\n",
      "17135165\n",
      "0.229601478081\n",
      "17136834\n",
      "0.220165661334\n",
      "17136978\n",
      "0.204926291534\n",
      "17135840\n",
      "0.181030884969\n",
      "16827698\n",
      "0.174920378348\n",
      "16026656\n",
      "0.166019536197\n",
      "16710006\n",
      "0.159519643985\n",
      "17134738\n",
      "0.157873346428\n",
      "16722702\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "from tinydb import TinyDB\n",
    "import ConfigParser\n",
    "import MySQLdb\n",
    "import json\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "import httplib\n",
    "import re \n",
    "import Stemmer\n",
    "\n",
    "config = ConfigParser.ConfigParser()\n",
    "config.readfp(open('my.cfg'))\n",
    "db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                     port=config.getint('mysqld', 'port'), \n",
    "                     user=config.get('mysqld', 'user'), \n",
    "                     passwd=config.get('mysqld', 'password'), \n",
    "                     db=config.get('mysqld', 'database') )\n",
    "db.set_character_set('utf8')\n",
    "cursor = db.cursor()\n",
    "cursor.execute('SET NAMES utf8;')\n",
    "cursor.execute('SET CHARACTER SET utf8;')\n",
    "cursor.execute('SET character_set_connection=utf8;')\n",
    "\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "spec_ids = pickle.load( open( \"spec_ids.p\", \"rb\" ) )\n",
    "key_skills = pickle.load( open( \"key_skills.p\", \"rb\" ) )\n",
    "title_words = pickle.load( open( \"title_words.p\", \"rb\" ) )\n",
    "\n",
    "count_vectorizer = pickle.load( open( \"count_vectorizer.p\", \"rb\" ) )\n",
    "tfidf_transformer = pickle.load( open( \"tfidf_transformer.p\", \"rb\" ) )\n",
    "\n",
    "def get_my_resume():\n",
    "    stemmer = Stemmer.Stemmer('russian')\n",
    "    headers = {\"User-Agent\": \"hh-recommender\", \"Authorization\" : \"Bearer T5MIT6GVV85LSVR75CB7U768TR3PFGS990I3QJNFV6A4CBQJF6M30G0MOT8U2V8I\"}\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    conn.request(\"GET\", \"https://api.hh.ru/resumes/mine\", headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "    me = r1.read()\n",
    "    me_json = json.loads(me)\n",
    "\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "#     conn.request(\"GET\", \"https://api.hh.ru/resumes/{}\".format(me_json['items'][0]['id']), headers=headers)\n",
    "    conn.request(\"GET\", \"https://api.hh.ru/resumes/{}\".format(\"5bdc936300031347c20039ed1f644868457044\"), headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "    resume = r1.read()\n",
    "    resume_json = json.loads(resume)\n",
    "\n",
    "    feature = []\n",
    "    resume_type = 'full'\n",
    "\n",
    "    resume_type = resume_json['employment']['id']\n",
    "\n",
    "    #description\n",
    "    p_doc = ''\n",
    "    doc = re.sub('<[^>]*>', '', resume_json['skills'].lower())\n",
    "    doc = re.sub('&quot;', '', doc)\n",
    "    doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', doc.strip())\n",
    "    for word in words:\n",
    "        word = stemmer.stemWord(word.strip())\n",
    "        if len(word.strip()) > 1:\n",
    "            p_doc = p_doc + \" \" + word\n",
    "\n",
    "    #title\n",
    "    p_title = ''\n",
    "    title = re.sub(ur'[^a-zа-я]+', ' ', resume_json['title'].lower(), re.UNICODE)\n",
    "    words = re.split(r'\\s{1,}', title.strip())\n",
    "    for title_word in words:\n",
    "        title_word = stemmer.stemWord(title_word)\n",
    "        if len(title_word.strip()) > 1:\n",
    "            p_title = p_title + \" \" + title_word.strip()\n",
    "\n",
    "    #keyskills\n",
    "    p_skills = ''\n",
    "    res_skills = resume_json['skill_set']\n",
    "    for skill in res_skills:\n",
    "        words = re.split(r'\\s{1,}', skill.lower().strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word)\n",
    "            if len(word.strip()) > 1:\n",
    "                p_skills = p_skills + \" \" + word.strip()\n",
    "                \n",
    "    #salary\n",
    "    salary = None\n",
    "    if resume_json['salary'] != None and resume_json['salary']['amount'] != None:\n",
    "        salary = resume_json['salary']['amount']/currency_rates[resume_json['salary']['currency']]\n",
    "    max_salary = 500000.0\n",
    "    if salary >= max_salary:\n",
    "        salary = max_salary\n",
    "\n",
    "    p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "    print p_doc\n",
    "    feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "    feature = tfidf_transformer.transform(feature_p_doc)\n",
    "    return feature.toarray(), salary\n",
    "\n",
    "\n",
    "def get_vacancies(offset, rows):\n",
    "    features = []\n",
    "\n",
    "    stemmer = Stemmer.Stemmer('russian')\n",
    "    cursor = db.cursor()\n",
    "    #будет задвоение, когда во время выборки в несколько запросом добавляются новые данные\n",
    "    cursor.execute(\"\"\"SELECT item, id FROM vacancies WHERE updated >= (NOW() - INTERVAL 5 DAY) LIMIT {}, {}\"\"\".format(offset, rows))\n",
    "    vacancy_ids = []\n",
    "    salaries = []\n",
    "    cities = []\n",
    "    for item in cursor:\n",
    "        feature = []\n",
    "        vacancy = json.loads(item[0])\n",
    "        vacancy_ids.append(vacancy['id'])\n",
    "\n",
    "        #description\n",
    "        p_doc = ''\n",
    "        doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "        doc = re.sub('&quot;', '', doc)\n",
    "        doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', doc.strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word.strip())\n",
    "            if len(word.strip()) > 1:\n",
    "                p_doc = p_doc + \" \" + word\n",
    "\n",
    "        #title\n",
    "        p_title = ''\n",
    "        title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', title.strip())\n",
    "        for title_word in words:\n",
    "            title_word = stemmer.stemWord(title_word)\n",
    "            if len(title_word.strip()) > 1:\n",
    "                p_title = p_title + \" \" + title_word.strip()\n",
    "\n",
    "        #keyskills\n",
    "        p_skills = ''\n",
    "        vac_skills = vacancy['key_skills']\n",
    "        for skill in vac_skills:\n",
    "            words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word)\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_skills = p_skills + \" \" + word.strip()\n",
    "                    \n",
    "        #salary\n",
    "        salary = None\n",
    "        if vacancy['salary'] != None:\n",
    "            if vacancy['salary']['from'] == None and vacancy['salary']['to'] != None:\n",
    "                salary = vacancy['salary']['to']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] == None and vacancy['salary']['from'] != None:\n",
    "                salary = vacancy['salary']['from']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] != None and vacancy['salary']['from'] != None:\n",
    "                salary = ((vacancy['salary']['from'] + vacancy['salary']['to'])/2)/currency_rates[vacancy['salary']['currency']]\n",
    "        max_salary = 500000.0\n",
    "        if salary >= max_salary:\n",
    "            salary = max_salary\n",
    "        salaries.append(salary)\n",
    "\n",
    "        p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "        \n",
    "\n",
    "        feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "        tfidf_feature_p_doc = tfidf_transformer.transform(feature_p_doc)\n",
    "        \n",
    "        cnt = 0\n",
    "        for f in tfidf_feature_p_doc.toarray()[0]:\n",
    "            if f > 0:\n",
    "                cnt = cnt + 1\n",
    "            \n",
    "        if cnt > 1:\n",
    "            features.append(tfidf_feature_p_doc.toarray()[0])\n",
    "\n",
    "    cursor.close()\n",
    "    return features, vacancy_ids, salaries\n",
    "\n",
    "def get_recommended(resume_feature, vacancy_features, resume_salary, vacancy_salaries, vacancy_ids):\n",
    "    new_vacancy_features = []\n",
    "    new_vacancy_ids = []\n",
    "    if resume_salary == None:\n",
    "        new_vacancy_features = vacancy_features\n",
    "        new_vacancy_ids = vacancy_ids\n",
    "    else:\n",
    "        i = 0\n",
    "        for vac_salary in vacancy_salaries:\n",
    "            if vac_salary == None:\n",
    "                new_vacancy_features.append(vacancy_features[i])\n",
    "                new_vacancy_ids.append(vacancy_ids[i])\n",
    "            else:\n",
    "                min_resume_salary = resume_salary - (resume_salary * 0.2)\n",
    "                max_resume_salary = resume_salary + (resume_salary * 0.8)\n",
    "                if vac_salary >= min_resume_salary and vac_salary <= max_resume_salary:\n",
    "                    new_vacancy_features.append(vacancy_features[i])\n",
    "                    new_vacancy_ids.append(vacancy_ids[i])\n",
    "                \n",
    "            i = i+1    \n",
    "    \n",
    "    if len(new_vacancy_features) > 0:\n",
    "        c_result = cosine_similarity(resume_feature, new_vacancy_features)\n",
    "        res = heapq.nlargest(15, range(len(c_result[0])), c_result[0].take)\n",
    "\n",
    "        similarities = []\n",
    "        ids = []\n",
    "        for j in res:\n",
    "            similarities.append(c_result[0][j])\n",
    "            ids.append(new_vacancy_ids[j])\n",
    "    return similarities, ids\n",
    "\n",
    "feature, salary = get_my_resume()\n",
    "\n",
    "count = 1000\n",
    "similarities = []\n",
    "ids = []\n",
    "features = get_vacancies(0, count)\n",
    "features, vacancy_ids, salaries = get_vacancies(0, count)\n",
    "\n",
    "f_len = len(features)\n",
    "\n",
    "r_similarities, r_ids = get_recommended(feature, features, salary, salaries, vacancy_ids)\n",
    "similarities = similarities + r_similarities\n",
    "ids = ids + r_ids\n",
    "\n",
    "i = 0\n",
    "while f_len > 0:\n",
    "    features, vacancy_ids, salaries = get_vacancies(i*count, count)\n",
    "    f_len = len(features)\n",
    "    if f_len > 0:\n",
    "        r_similarities, r_ids = get_recommended(feature, features, salary, salaries, vacancy_ids)\n",
    "        similarities = similarities + r_similarities\n",
    "        ids = ids + r_ids\n",
    "    \n",
    "    i = i+1\n",
    "    print 'processed {} vаcancies'.format(i*count)\n",
    "        \n",
    "    if i == 10:\n",
    "        break\n",
    "        \n",
    "max_similarities = heapq.nlargest(15, range(len(numpy.asarray(similarities))), numpy.asarray(similarities).take)\n",
    "for ind in max_similarities:\n",
    "    print similarities[ind]\n",
    "    print ids[ind]\n",
    "    \n",
    "db.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start at 2016-06-01 17:13:25.445153\n",
      "processed 1000 vаcancies\n",
      "processed 2000 vаcancies\n",
      "processed 3000 vаcancies\n",
      "processed 4000 vаcancies\n",
      "processed 5000 vаcancies\n",
      "processed 6000 vаcancies\n",
      "processed 7000 vаcancies\n",
      "processed 8000 vаcancies\n",
      "processed 9000 vаcancies\n",
      "processed 10000 vаcancies\n",
      "processed 11000 vаcancies\n",
      "processed 12000 vаcancies\n",
      "processed 13000 vаcancies\n",
      "processed 14000 vаcancies\n",
      "processed 15000 vаcancies\n",
      "processed 16000 vаcancies\n",
      "processed 17000 vаcancies\n",
      "processed 18000 vаcancies\n",
      "processed 19000 vаcancies\n",
      "processed 20000 vаcancies\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a\n",
      "for 17185733 similarity is 0.241420428222\n",
      "for 16858436 similarity is 0.231747031736\n",
      "for 15685222 similarity is 0.217491396879\n",
      "for 17098616 similarity is 0.213119123972\n",
      "for 16825405 similarity is 0.209957690914\n",
      "for 17197588 similarity is 0.201732166924\n",
      "for 17193832 similarity is 0.197981865485\n",
      "for 16165915 similarity is 0.195952985719\n",
      "for 16716632 similarity is 0.193102134053\n",
      "for 17190074 similarity is 0.190184213484\n",
      "for 17188217 similarity is 0.189807166359\n",
      "for 17190141 similarity is 0.183846894776\n",
      "for 15045209 similarity is 0.182566398696\n",
      "for 17124148 similarity is 0.179818133491\n",
      "for 15045200 similarity is 0.177550143234\n",
      "for 17182223 similarity is 0.17651011305\n",
      "for 17182223 similarity is 0.17651011305\n",
      "for 15431158 similarity is 0.170411555353\n",
      "for 16679291 similarity is 0.161486448948\n",
      "for 16711861 similarity is 0.151652867442\n",
      "4527975dff02f59d5f0039ed1f534356744c6e\n",
      "for 17190592 similarity is 0.32853290184\n",
      "for 16803389 similarity is 0.308989905585\n",
      "for 15685222 similarity is 0.297546103535\n",
      "for 17182094 similarity is 0.288522303422\n",
      "for 17182094 similarity is 0.288522303422\n",
      "for 17014281 similarity is 0.277741825232\n",
      "for 17185733 similarity is 0.27079151884\n",
      "for 17190074 similarity is 0.244443850617\n",
      "for 17182193 similarity is 0.243885821073\n",
      "for 17182193 similarity is 0.243885821073\n",
      "for 16825405 similarity is 0.243197307423\n",
      "for 16585330 similarity is 0.239942405205\n",
      "for 17182830 similarity is 0.225159026085\n",
      "for 17182830 similarity is 0.225159026085\n",
      "for 17183677 similarity is 0.224284520785\n",
      "for 16858436 similarity is 0.218734527172\n",
      "for 16683078 similarity is 0.216280094867\n",
      "for 17183052 similarity is 0.207465712192\n",
      "for 16165915 similarity is 0.198576857039\n",
      "for 17187549 similarity is 0.1932426447\n",
      "848f3de0ff000c40da0039ed1f736563726574\n",
      "for 17185733 similarity is 0.261828800225\n",
      "for 16858436 similarity is 0.242504778706\n",
      "for 17098616 similarity is 0.228706829185\n",
      "for 15685222 similarity is 0.227540897071\n",
      "for 16165915 similarity is 0.222043158323\n",
      "for 16825405 similarity is 0.22128041777\n",
      "for 17190141 similarity is 0.200561978596\n",
      "for 17190074 similarity is 0.198996694012\n",
      "for 17197588 similarity is 0.198967574545\n",
      "for 17188217 similarity is 0.197002169865\n",
      "for 17193832 similarity is 0.195268668255\n",
      "for 16716632 similarity is 0.194427885552\n",
      "for 17124148 similarity is 0.191434033214\n",
      "for 15431158 similarity is 0.183046982862\n",
      "for 15045209 similarity is 0.180064459207\n",
      "for 16711861 similarity is 0.178968946636\n",
      "for 15045200 similarity is 0.175116947872\n",
      "for 17182223 similarity is 0.174091170544\n",
      "for 17182223 similarity is 0.174091170544\n",
      "for 17193643 similarity is 0.171526618719\n",
      "04589be6ff01f2fef60039ed1f56534e424e42\n",
      "for 17196267 similarity is 0.347377411128\n",
      "for 17183846 similarity is 0.31507728119\n",
      "for 17194918 similarity is 0.259246490004\n",
      "for 17198776 similarity is 0.249693840602\n",
      "for 17196960 similarity is 0.193201172172\n",
      "for 16476271 similarity is 0.171531032001\n",
      "for 17052776 similarity is 0.168706725035\n",
      "for 17188895 similarity is 0.15081751129\n",
      "for 17188748 similarity is 0.14584607867\n",
      "for 17184427 similarity is 0.141031629954\n",
      "for 17192323 similarity is 0.140576082179\n",
      "for 17182928 similarity is 0.125456917786\n",
      "for 17193953 similarity is 0.124317609046\n",
      "for 17198486 similarity is 0.122805150635\n",
      "for 17182648 similarity is 0.121051896874\n",
      "for 17182648 similarity is 0.121051896874\n",
      "for 17182688 similarity is 0.117621283548\n",
      "for 17182688 similarity is 0.117621283548\n",
      "for 17192040 similarity is 0.112092600228\n",
      "for 16583323 similarity is 0.110703442662\n",
      "a588f755ff02418ca00039ed1f5a75387a3955\n",
      "for 17181878 similarity is 0.0\n",
      "for 17181879 similarity is 0.0\n",
      "for 17181881 similarity is 0.0\n",
      "for 17181892 similarity is 0.0\n",
      "for 17181894 similarity is 0.0\n",
      "for 17181908 similarity is 0.0\n",
      "for 17181911 similarity is 0.0\n",
      "for 16941968 similarity is 0.0\n",
      "for 17181916 similarity is 0.0\n",
      "for 17181924 similarity is 0.0\n",
      "for 17181925 similarity is 0.0\n",
      "for 17181938 similarity is 0.0\n",
      "for 17181944 similarity is 0.0\n",
      "for 17181949 similarity is 0.0\n",
      "for 17181961 similarity is 0.0\n",
      "for 17181967 similarity is 0.0\n",
      "for 17181972 similarity is 0.0\n",
      "for 14690439 similarity is 0.0\n",
      "for 17181974 similarity is 0.0\n",
      "for 17181980 similarity is 0.0\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642\n",
      "for 16977650 similarity is 0.328960023028\n",
      "for 17191425 similarity is 0.303407733652\n",
      "for 16895056 similarity is 0.295275719024\n",
      "for 17187260 similarity is 0.252307966964\n",
      "for 17182323 similarity is 0.241233208258\n",
      "for 17182323 similarity is 0.241233208258\n",
      "for 17192228 similarity is 0.239448984579\n",
      "for 16873362 similarity is 0.232595859938\n",
      "for 17192930 similarity is 0.218048261794\n",
      "for 16898603 similarity is 0.209261470205\n",
      "for 17190491 similarity is 0.194549025145\n",
      "for 17183024 similarity is 0.184081680497\n",
      "for 17187244 similarity is 0.162514360595\n",
      "for 17066755 similarity is 0.160708895232\n",
      "for 15065374 similarity is 0.155506579317\n",
      "for 15065375 similarity is 0.155506579317\n",
      "for 16432831 similarity is 0.154820607105\n",
      "for 17190667 similarity is 0.154806479736\n",
      "for 16976331 similarity is 0.153476546131\n",
      "for 17189731 similarity is 0.15340405254\n",
      "27cf52c1ff00b6dc6d0039ed1f736563726574\n",
      "for 15431158 similarity is 0.362254145256\n",
      "for 17124148 similarity is 0.29410149051\n",
      "for 15685222 similarity is 0.289767363695\n",
      "for 17182094 similarity is 0.271701788903\n",
      "for 17182094 similarity is 0.271701788903\n",
      "for 17182830 similarity is 0.258590119731\n",
      "for 17182830 similarity is 0.258590119731\n",
      "for 16825405 similarity is 0.255158467325\n",
      "for 16165915 similarity is 0.246060346584\n",
      "for 17193561 similarity is 0.244257352046\n",
      "for 17185733 similarity is 0.233767330032\n",
      "for 16803389 similarity is 0.224634473624\n",
      "for 16679291 similarity is 0.218911853448\n",
      "for 17182193 similarity is 0.211796680403\n",
      "for 17182193 similarity is 0.211796680403\n",
      "for 14121053 similarity is 0.210809709834\n",
      "for 17182643 similarity is 0.210268995969\n",
      "for 17182643 similarity is 0.210268995969\n",
      "for 17190592 similarity is 0.209984133237\n",
      "for 17183052 similarity is 0.197427628001\n",
      "total time 117.014307022 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "from tinydb import TinyDB\n",
    "import ConfigParser\n",
    "import MySQLdb\n",
    "import json\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "import httplib\n",
    "import re \n",
    "import Stemmer\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "print 'Start at {}'.format(datetime.datetime.now())\n",
    "start_time = time.time()\n",
    "config = ConfigParser.ConfigParser()\n",
    "config.readfp(open('my.cfg'))\n",
    "db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                     port=config.getint('mysqld', 'port'), \n",
    "                     user=config.get('mysqld', 'user'), \n",
    "                     passwd=config.get('mysqld', 'password'), \n",
    "                     db=config.get('mysqld', 'database') )\n",
    "db.set_character_set('utf8')\n",
    "cursor = db.cursor()\n",
    "cursor.execute('SET NAMES utf8;')\n",
    "cursor.execute('SET CHARACTER SET utf8;')\n",
    "cursor.execute('SET character_set_connection=utf8;')\n",
    "\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "#areas\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/areas\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "areas = r1.read()\n",
    "areas_json = json.loads(areas)\n",
    "areas_map = {}\n",
    "def build_areas_map(areas, areas_map):\n",
    "    for area in areas:\n",
    "        if area['id'] == '1':#msk\n",
    "            parent_id = '2019'\n",
    "        elif area['id'] == '2':#spb\n",
    "            parent_id = '145'\n",
    "        elif area['id'] == '115':#kiev\n",
    "            parent_id = '2164'\n",
    "        elif area['id'] == '1002':#minsk\n",
    "            parent_id = '2237'\n",
    "        else:\n",
    "            parent_id = area['parent_id']\n",
    "        areas_map[area['id']] = parent_id\n",
    "        build_areas_map(area['areas'], areas_map)\n",
    "        \n",
    "build_areas_map(areas_json, areas_map)\n",
    "    \n",
    "spec_ids = pickle.load( open( \"spec_ids.p\", \"rb\" ) )\n",
    "key_skills = pickle.load( open( \"key_skills.p\", \"rb\" ) )\n",
    "title_words = pickle.load( open( \"title_words.p\", \"rb\" ) )\n",
    "\n",
    "count_vectorizer = pickle.load( open( \"count_vectorizer.p\", \"rb\" ) )\n",
    "tfidf_transformer = pickle.load( open( \"tfidf_transformer.p\", \"rb\" ) )\n",
    "\n",
    "def get_resumes():\n",
    "    salaries = []\n",
    "    features = []\n",
    "    ids = []\n",
    "    areas = []\n",
    "    stemmer = Stemmer.Stemmer('russian')\n",
    "    cursor = db.cursor()\n",
    "    cursor.execute(\"\"\"SELECT item FROM resumes WHERE is_active=1\"\"\")\n",
    "    for item in cursor:\n",
    "        resume_json = json.loads(item[0])\n",
    "        feature = []\n",
    "        #description\n",
    "        p_doc = ''\n",
    "        if resume_json['skills'] != None:\n",
    "            doc = re.sub('<[^>]*>', '', resume_json['skills'].lower())\n",
    "            doc = re.sub('&quot;', '', doc)\n",
    "            doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "            words = re.split(r'\\s{1,}', doc.strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word.strip())\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_doc = p_doc + \" \" + word\n",
    "\n",
    "        #title\n",
    "        p_title = ''\n",
    "        if resume_json['title'] != None:\n",
    "            title = re.sub(ur'[^a-zа-я]+', ' ', resume_json['title'].lower(), re.UNICODE)\n",
    "            words = re.split(r'\\s{1,}', title.strip())\n",
    "            for title_word in words:\n",
    "                title_word = stemmer.stemWord(title_word)\n",
    "                if len(title_word.strip()) > 1:\n",
    "                    p_title = p_title + \" \" + title_word.strip()\n",
    "\n",
    "        #keyskills\n",
    "        p_skills = ''\n",
    "        res_skills = resume_json['skill_set']\n",
    "        for skill in res_skills:\n",
    "            words = re.split(r'\\s{1,}', skill.lower().strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word)\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_skills = p_skills + \" \" + word.strip()\n",
    "\n",
    "        #salary\n",
    "        salary = None\n",
    "        if resume_json['salary'] != None and resume_json['salary']['amount'] != None:\n",
    "            salary = resume_json['salary']['amount']/currency_rates[resume_json['salary']['currency']]\n",
    "        max_salary = 500000.0\n",
    "        if salary >= max_salary:\n",
    "            salary = max_salary\n",
    "        \n",
    "        \n",
    "        res_areas = []\n",
    "        if resume_json['area'] == None:\n",
    "            res_areas.append(areas_map[\"1\"])\n",
    "        else :\n",
    "            res_areas.append(areas_map[resume_json['area']['id']])\n",
    "        for area in resume_json['relocation']['area']:\n",
    "            res_areas.append(areas_map[area['id']])\n",
    "        areas.append(res_areas)\n",
    "        \n",
    "\n",
    "        p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "        feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "        feature = tfidf_transformer.transform(feature_p_doc)\n",
    "        features.append(feature.toarray())\n",
    "        salaries.append(salary)\n",
    "        ids.append(resume_json['id'])\n",
    "    cursor.close()\n",
    "    return features, salaries, ids, areas\n",
    "\n",
    "\n",
    "def get_vacancies(offset, rows):\n",
    "    features = []\n",
    "\n",
    "    stemmer = Stemmer.Stemmer('russian')\n",
    "    cursor = db.cursor()\n",
    "    #будет задвоение, когда во время выборки в несколько запросом добавляются новые данные\n",
    "    cursor.execute(\"\"\"SELECT item, id FROM vacancies WHERE updated >= (NOW() - INTERVAL 7 DAY) LIMIT {}, {}\"\"\".format(offset, rows))\n",
    "    vacancy_ids = []\n",
    "    salaries = []\n",
    "    cities = []\n",
    "    titles = []\n",
    "    areas = []\n",
    "    for item in cursor:\n",
    "        feature = []\n",
    "        vacancy = json.loads(item[0])\n",
    "        vacancy_ids.append(vacancy['id'])\n",
    "\n",
    "        #description\n",
    "        p_doc = ''\n",
    "        doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "        doc = re.sub('&quot;', '', doc)\n",
    "        doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', doc.strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word.strip())\n",
    "            if len(word.strip()) > 1:\n",
    "                p_doc = p_doc + \" \" + word\n",
    "\n",
    "        #title\n",
    "        p_title = ''\n",
    "        title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', title.strip())\n",
    "        for title_word in words:\n",
    "            title_word = stemmer.stemWord(title_word)\n",
    "            if len(title_word.strip()) > 1:\n",
    "                p_title = p_title + \" \" + title_word.strip()\n",
    "                \n",
    "        titles.append(vacancy['name'])\n",
    "\n",
    "        #keyskills\n",
    "        p_skills = ''\n",
    "        vac_skills = vacancy['key_skills']\n",
    "        for skill in vac_skills:\n",
    "            words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word)\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_skills = p_skills + \" \" + word.strip()\n",
    "                    \n",
    "        #salary\n",
    "        salary = None\n",
    "        if vacancy['salary'] != None:\n",
    "            if vacancy['salary']['from'] == None and vacancy['salary']['to'] != None:\n",
    "                salary = vacancy['salary']['to']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] == None and vacancy['salary']['from'] != None:\n",
    "                salary = vacancy['salary']['from']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] != None and vacancy['salary']['from'] != None:\n",
    "                salary = ((vacancy['salary']['from'] + vacancy['salary']['to'])/2)/currency_rates[vacancy['salary']['currency']]\n",
    "        max_salary = 500000.0\n",
    "        if salary >= max_salary:\n",
    "            salary = max_salary\n",
    "        salaries.append(salary)\n",
    "        try:\n",
    "            areas.append(areas_map[vacancy['area']['id']])\n",
    "        except KeyError:\n",
    "            print 'missed area id {} for vacancy {}'.format(vacancy['area']['id'], vacancy['id'])\n",
    "            \n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/vacancies/{}\".format(vacancy['id']), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "            missed_area_vacancy = r1.read()\n",
    "            missed_area_vacancy_json = json.loads(missed_area_vacancy)\n",
    "            \n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/areas/{}\"\n",
    "                         .format(missed_area_vacancy_json['area']['id']), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "            missed_area = r1.read()\n",
    "            missed_area_json = json.loads(missed_area)\n",
    "            areas_map[vacancy['area']['id']] = missed_area_json['parent_id']\n",
    "            areas.append(missed_area_json['parent_id'])\n",
    "\n",
    "        p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "        \n",
    "\n",
    "        feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "        tfidf_feature_p_doc = tfidf_transformer.transform(feature_p_doc)\n",
    "            \n",
    "        features.append(tfidf_feature_p_doc.toarray()[0])\n",
    "\n",
    "    cursor.close()\n",
    "    return features, vacancy_ids, salaries, titles, areas\n",
    "\n",
    "def get_recommended(resume_feature, vacancy_features, resume_salary, vacancy_salaries, vacancy_ids, vac_titles, resume_areas, vacancies_area):\n",
    "    pre_vacancy_features = []\n",
    "    pre_vacancy_ids = []\n",
    "    pre_vac_titles = []\n",
    "    pre_vacancy_salaries = []\n",
    "    j = 0\n",
    "    for vac_area in vacancies_area:\n",
    "        if vac_area in resume_areas:\n",
    "            pre_vacancy_features.append(vacancy_features[j])\n",
    "            pre_vacancy_ids.append(vacancy_ids[j])\n",
    "            pre_vac_titles.append(vac_titles[j])\n",
    "            pre_vacancy_salaries.append(vacancy_salaries[j])\n",
    "        j = j+1\n",
    "        \n",
    "    new_vacancy_features = []\n",
    "    new_vacancy_ids = []\n",
    "    new_vac_titles = []\n",
    "    if resume_salary == None:\n",
    "        new_vacancy_features = pre_vacancy_features\n",
    "        new_vacancy_ids = pre_vacancy_ids\n",
    "        new_vac_titles = pre_vac_titles\n",
    "    else:\n",
    "        i = 0\n",
    "        for vac_salary in pre_vacancy_salaries:\n",
    "            if vac_salary == None:\n",
    "                new_vacancy_features.append(pre_vacancy_features[i])\n",
    "                new_vacancy_ids.append(pre_vacancy_ids[i])\n",
    "                new_vac_titles.append(pre_vac_titles[i])\n",
    "            else:\n",
    "                min_resume_salary = resume_salary - (resume_salary * 0.2)\n",
    "                max_resume_salary = resume_salary + (resume_salary * 0.8)\n",
    "                if vac_salary >= min_resume_salary and vac_salary <= max_resume_salary:\n",
    "                    new_vacancy_features.append(pre_vacancy_features[i])\n",
    "                    new_vacancy_ids.append(pre_vacancy_ids[i])\n",
    "                    new_vac_titles.append(pre_vac_titles[i])\n",
    "                \n",
    "            i = i+1    \n",
    "    \n",
    "    similarities = []\n",
    "    ids = []\n",
    "    titles = []\n",
    "    if len(new_vacancy_features) > 0:\n",
    "        c_result = cosine_similarity(resume_feature, new_vacancy_features)\n",
    "        res = heapq.nlargest(20, range(len(c_result[0])), c_result[0].take)\n",
    "        \n",
    "        for j in res:\n",
    "            similarities.append(c_result[0][j])\n",
    "            ids.append(new_vacancy_ids[j])\n",
    "            titles.append(new_vac_titles[j])\n",
    "    return similarities, ids, titles\n",
    "\n",
    "resume_features, resume_salaries, resume_ids, resume_areas = get_resumes()\n",
    "\n",
    "count = 1000\n",
    "features = get_vacancies(0, count)\n",
    "features, vacancy_ids, salaries, titles, vacancy_areas = get_vacancies(0, count)\n",
    "\n",
    "f_len = len(features)\n",
    "\n",
    "res_similarities = {}\n",
    "res_recommended_ids = {}\n",
    "res_recommended_titles = {}\n",
    "for idx, val in enumerate(resume_features):  \n",
    "    r_similarities, r_ids, r_titles = get_recommended(resume_features[idx], features, resume_salaries[idx], salaries, vacancy_ids, \n",
    "                                                      titles, resume_areas[idx], vacancy_areas)\n",
    "    res_similarities[resume_ids[idx]] = r_similarities\n",
    "    res_recommended_ids[resume_ids[idx]] = r_ids\n",
    "    res_recommended_titles[resume_ids[idx]] = r_titles\n",
    "\n",
    "i = 0\n",
    "while f_len > 0:\n",
    "    features, vacancy_ids, salaries, titles, vacancy_areas = get_vacancies(i*count, count)\n",
    "    f_len = len(features)\n",
    "    if f_len > 0:\n",
    "        for idx, val in enumerate(resume_features):\n",
    "            r_similarities, r_ids, r_titles = get_recommended(resume_features[idx], features, resume_salaries[idx], salaries, vacancy_ids, \n",
    "                                                              titles, resume_areas[idx], vacancy_areas)\n",
    "            res_similarities[resume_ids[idx]] = res_similarities[resume_ids[idx]] + r_similarities\n",
    "            res_recommended_ids[resume_ids[idx]] = res_recommended_ids[resume_ids[idx]] + r_ids\n",
    "            res_recommended_titles[resume_ids[idx]] = res_recommended_titles[resume_ids[idx]] + r_titles\n",
    "            \n",
    "    i = i+1\n",
    "    print 'processed {} vаcancies'.format(i*count)\n",
    "        \n",
    "    if i == 20:\n",
    "        break\n",
    "\n",
    "for resume_id in res_similarities.keys():\n",
    "    print resume_id\n",
    "    similarities = res_similarities[resume_id]\n",
    "    ids = res_recommended_ids[resume_id]\n",
    "    titles = res_recommended_titles[resume_id]\n",
    "    max_similarities = heapq.nlargest(20, range(len(numpy.asarray(similarities))), numpy.asarray(similarities).take)\n",
    "    cursor = db.cursor()\n",
    "    try:\n",
    "        cursor.execute(\"\"\"UPDATE recommendations SET is_active=0 WHERE resume_id='{}'\"\"\".format(resume_id))\n",
    "    except BaseException:\n",
    "        db.rollback()\n",
    "    finally:\n",
    "        cursor.close()\n",
    "    for ind in max_similarities:\n",
    "        cursor = db.cursor()\n",
    "        try:\n",
    "            cursor.execute(\"\"\"INSERT INTO recommendations (resume_id, vacancy_id, updated, is_active, similarity, vacancy_title) VALUES ('{}', {}, now(), 1, {}, '{}')\"\"\".format(resume_id, ids[ind], similarities[ind], titles[ind].encode('utf-8').strip()))\n",
    "        except BaseException:\n",
    "            db.rollback()\n",
    "        finally:\n",
    "            cursor.close()\n",
    "        print 'for {} similarity is {}'.format(ids[ind], similarities[ind])\n",
    "    db.commit()\n",
    "        \n",
    "db.commit()\n",
    "db.close()\n",
    "\n",
    "print 'total time {} sec\\n'.format(time.time()-start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'bar'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import redis\n",
    "r = redis.StrictRedis(host='localhost', port=6379, db=0)\n",
    "r.set('foo', 'bar')\n",
    "r.get('foo')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start at 2016-06-06 16:25:07.609814\n",
      "loaded 2000\n",
      "loaded 4000\n",
      "loaded 6000\n",
      "loaded 8000\n",
      "loaded 10000\n",
      "missed area id 3309 for vacancy 15891681\n",
      "loaded 12000\n",
      "loaded 14000\n",
      "loaded 16000\n",
      "loaded 18000\n",
      "loaded 20000\n",
      "loaded 22000\n",
      "loaded 24000\n",
      "loaded 26000\n",
      "loaded 28000\n",
      "loaded 30000\n",
      "loaded 32000\n",
      "loaded 34000\n",
      "loaded 36000\n",
      "loaded 38000\n",
      "loaded 40000\n",
      "loaded 42000\n",
      "loaded 44000\n",
      "loaded 46000\n",
      "loaded 48000\n",
      "loaded 50000\n",
      "loaded 52000\n",
      "loaded 54000\n",
      "loaded 56000\n",
      "loaded 58000\n",
      "loaded 60000\n",
      "loaded 62000\n",
      "loaded 64000\n",
      "loaded 66000\n",
      "loaded 68000\n",
      "loaded 70000\n",
      "loaded 72000\n",
      "loaded 74000\n",
      "loaded 76000\n",
      "loaded 78000\n",
      "loaded 80000\n",
      "loaded 82000\n",
      "loaded 84000\n",
      "loaded 86000\n",
      "loaded 88000\n",
      "loaded 90000\n",
      "loaded 92000\n",
      "loaded 94000\n",
      "loaded 96000\n",
      "loaded 98000\n",
      "loaded 100000\n",
      "loaded 102000\n",
      "loaded 104000\n",
      "loaded 106000\n",
      "loaded 108000\n",
      "loaded 110000\n",
      "loaded 112000\n",
      "loaded 114000\n",
      "loaded 116000\n",
      "loaded 118000\n",
      "loaded 120000\n",
      "loaded 122000\n",
      "loaded 124000\n",
      "loaded 126000\n",
      "loaded 128000\n",
      "loaded 130000\n",
      "loaded 132000\n",
      "loaded 134000\n",
      "loaded 136000\n",
      "loaded 138000\n",
      "loaded 140000\n",
      "loaded 142000\n",
      "loaded 144000\n",
      "loaded 146000\n",
      "loaded 148000\n",
      "loaded 150000\n",
      "loaded 152000\n",
      "finish  at 34.5748804132 min\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "# save all vacancies into redis\n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "from tinydb import TinyDB\n",
    "import ConfigParser\n",
    "import MySQLdb\n",
    "import json\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from scipy import spatial\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import heapq\n",
    "import numpy\n",
    "import httplib\n",
    "import re \n",
    "import Stemmer\n",
    "import time\n",
    "import datetime\n",
    "import redis\n",
    "\n",
    "print 'Start at {}'.format(datetime.datetime.now())\n",
    "r = redis.StrictRedis(host='localhost', port=6379, db=0)\n",
    "start_time = time.time()\n",
    "config = ConfigParser.ConfigParser()\n",
    "config.readfp(open('my.cfg'))\n",
    "db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                     port=config.getint('mysqld', 'port'), \n",
    "                     user=config.get('mysqld', 'user'), \n",
    "                     passwd=config.get('mysqld', 'password'), \n",
    "                     db=config.get('mysqld', 'database') )\n",
    "db.set_character_set('utf8')\n",
    "cursor = db.cursor()\n",
    "cursor.execute('SET NAMES utf8;')\n",
    "cursor.execute('SET CHARACTER SET utf8;')\n",
    "cursor.execute('SET character_set_connection=utf8;')\n",
    "\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "#areas\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/areas\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "areas = r1.read()\n",
    "areas_json = json.loads(areas)\n",
    "areas_map = {}\n",
    "def build_areas_map(areas, areas_map):\n",
    "    for area in areas:\n",
    "        if area['id'] == '1':#msk\n",
    "            parent_id = '2019'\n",
    "        elif area['id'] == '2':#spb\n",
    "            parent_id = '145'\n",
    "        elif area['id'] == '115':#kiev\n",
    "            parent_id = '2164'\n",
    "        elif area['id'] == '1002':#minsk\n",
    "            parent_id = '2237'\n",
    "        else:\n",
    "            parent_id = area['parent_id']\n",
    "        areas_map[area['id']] = parent_id\n",
    "        build_areas_map(area['areas'], areas_map)\n",
    "        \n",
    "build_areas_map(areas_json, areas_map)\n",
    "    \n",
    "spec_ids = pickle.load( open( \"spec_ids.p\", \"rb\" ) )\n",
    "key_skills = pickle.load( open( \"key_skills.p\", \"rb\" ) )\n",
    "title_words = pickle.load( open( \"title_words.p\", \"rb\" ) )\n",
    "\n",
    "count_vectorizer = pickle.load( open( \"count_vectorizer.p\", \"rb\" ) )\n",
    "tfidf_transformer = pickle.load( open( \"tfidf_transformer.p\", \"rb\" ) )\n",
    "\n",
    "def get_vacancies(offset, rows):\n",
    "    features = []\n",
    "\n",
    "    stemmer = Stemmer.Stemmer('russian')\n",
    "    cursor = db.cursor()\n",
    "    #будет задвоение, когда во время выборки в несколько запросом добавляются новые данные\n",
    "    cursor.execute(\"\"\"SELECT item, id FROM vacancies WHERE updated >= (NOW() - INTERVAL 7 DAY) LIMIT {}, {}\"\"\".format(offset, rows))\n",
    "    vacancy_ids = []\n",
    "    salaries = []\n",
    "    cities = []\n",
    "    titles = []\n",
    "    areas = []\n",
    "    for item in cursor:\n",
    "        feature = []\n",
    "        vacancy = json.loads(item[0])\n",
    "        vacancy_ids.append(vacancy['id'])\n",
    "\n",
    "        #description\n",
    "        p_doc = ''\n",
    "        doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "        doc = re.sub('&quot;', '', doc)\n",
    "        doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', doc.strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word.strip())\n",
    "            if len(word.strip()) > 1:\n",
    "                p_doc = p_doc + \" \" + word\n",
    "\n",
    "        #title\n",
    "        p_title = ''\n",
    "        title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', title.strip())\n",
    "        for title_word in words:\n",
    "            title_word = stemmer.stemWord(title_word)\n",
    "            if len(title_word.strip()) > 1:\n",
    "                p_title = p_title + \" \" + title_word.strip()\n",
    "                \n",
    "        titles.append(vacancy['name'])\n",
    "\n",
    "        #keyskills\n",
    "        p_skills = ''\n",
    "        vac_skills = vacancy['key_skills']\n",
    "        for skill in vac_skills:\n",
    "            words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word)\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_skills = p_skills + \" \" + word.strip()\n",
    "                    \n",
    "        #salary\n",
    "        salary = None\n",
    "        if vacancy['salary'] != None:\n",
    "            if vacancy['salary']['from'] == None and vacancy['salary']['to'] != None:\n",
    "                salary = vacancy['salary']['to']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] == None and vacancy['salary']['from'] != None:\n",
    "                salary = vacancy['salary']['from']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] != None and vacancy['salary']['from'] != None:\n",
    "                salary = ((vacancy['salary']['from'] + vacancy['salary']['to'])/2)/currency_rates[vacancy['salary']['currency']]\n",
    "        max_salary = 500000.0\n",
    "        if salary >= max_salary:\n",
    "            salary = max_salary\n",
    "        salaries.append(salary)\n",
    "        try:\n",
    "            areas.append(areas_map[vacancy['area']['id']])\n",
    "        except KeyError:\n",
    "            print 'missed area id {} for vacancy {}'.format(vacancy['area']['id'], vacancy['id'])\n",
    "            \n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/vacancies/{}\".format(vacancy['id']), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "            missed_area_vacancy = r1.read()\n",
    "            missed_area_vacancy_json = json.loads(missed_area_vacancy)\n",
    "            \n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/areas/{}\"\n",
    "                         .format(missed_area_vacancy_json['area']['id']), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "            missed_area = r1.read()\n",
    "            missed_area_json = json.loads(missed_area)\n",
    "            areas_map[vacancy['area']['id']] = missed_area_json['parent_id']\n",
    "            areas.append(missed_area_json['parent_id'])\n",
    "\n",
    "        p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "        \n",
    "\n",
    "        feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "        tfidf_feature_p_doc = tfidf_transformer.transform(feature_p_doc)\n",
    "            \n",
    "        features.append(tfidf_feature_p_doc.toarray()[0])\n",
    "\n",
    "    cursor.close()\n",
    "    return features, vacancy_ids, salaries, titles, areas\n",
    "\n",
    "timeout = 6*24*60*60\n",
    "vac_cnt = 2000\n",
    "\n",
    "features, vacancy_ids, salaries, titles, areas = get_vacancies(0, vac_cnt)\n",
    "cnt = len(features)\n",
    "for idx, val in enumerate(features): \n",
    "    data = {}\n",
    "    data['features'] = json.dumps(features[idx].tolist()).encode(\"zlib\")\n",
    "    data['salary'] = salaries[idx]\n",
    "    data['area'] = areas[idx]\n",
    "    r.hmset(vacancy_ids[idx], data)\n",
    "    r.expire(vacancy_ids[idx], timeout)\n",
    "    \n",
    "i = 0\n",
    "while cnt > 0:\n",
    "    features, vacancy_ids, salaries, titles, areas = get_vacancies(i*vac_cnt, vac_cnt)\n",
    "    cnt = len(features)\n",
    "    for idx, val in enumerate(features): \n",
    "        data = {}\n",
    "        data['features'] = json.dumps(features[idx].tolist()).encode(\"zlib\")\n",
    "        data['salary'] = salaries[idx]\n",
    "        data['area'] = areas[idx]\n",
    "        r.hmset(vacancy_ids[idx], data)\n",
    "        r.expire(vacancy_ids[idx], timeout)\n",
    "    print 'loaded {}'.format(i*vac_cnt+vac_cnt)\n",
    "    i = i+1\n",
    "\n",
    "    \n",
    "print \"finish  at {} min\".format((time.time()-start_time)/60.0)\n",
    "\n",
    "#print data['features'].decode('zlib')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start at 2016-06-22 14:44:27.766073\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:39:28&date_to=2016-06-22T14:44:28&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:39:28&date_to=2016-06-22T14:44:28&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:39:28&date_to=2016-06-22T14:44:28&page=2\n",
      "410\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "47.0739178658 sec\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "#vacancy downloader 2\n",
    "import httplib\n",
    "import json\n",
    "import time\n",
    "import threading\n",
    "import pickle\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import heapq\n",
    "import numpy\n",
    "import re \n",
    "import Stemmer\n",
    "import datetime\n",
    "import redis\n",
    "\n",
    "\n",
    "print 'Start at {}'.format(datetime.datetime.now())\n",
    "r = redis.StrictRedis(host='localhost', port=6379, db=0)\n",
    "start_time = time.time()\n",
    "timeout = 5*24*60*60\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "if r1.status != 200:\n",
    "    conn.close()\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "#areas\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/areas\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "if r1.status != 200:\n",
    "    conn.close()\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    conn.request(\"GET\", \"https://api.hh.ru/areas\", headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "areas = r1.read()\n",
    "conn.close()\n",
    "areas_json = json.loads(areas)\n",
    "areas_map = {}\n",
    "def build_areas_map(areas, areas_map):\n",
    "    for area in areas:\n",
    "        if area['id'] == '1':#msk\n",
    "            parent_id = '2019'\n",
    "        elif area['id'] == '2':#spb\n",
    "            parent_id = '145'\n",
    "        elif area['id'] == '115':#kiev\n",
    "            parent_id = '2164'\n",
    "        elif area['id'] == '1002':#minsk\n",
    "            parent_id = '2237'\n",
    "        else:\n",
    "            parent_id = area['parent_id']\n",
    "        areas_map[area['id']] = parent_id\n",
    "        build_areas_map(area['areas'], areas_map)\n",
    "        \n",
    "build_areas_map(areas_json, areas_map)\n",
    "    \n",
    "with open( \"count_vectorizer.p\", \"rb\" ) as f:\n",
    "    count_vectorizer = pickle.load(f)\n",
    "    \n",
    "with open( \"tfidf_transformer.p\", \"rb\" ) as f:\n",
    "    tfidf_transformer = pickle.load(f)\n",
    "    \n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "    \n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "\n",
    "def get_vacancy_ids():\n",
    "    vacancy_ids = []\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    per_page = 200\n",
    "    page = 0\n",
    "    count = per_page\n",
    "    date_from = (datetime.datetime.now() - datetime.timedelta(minutes=5)).strftime('%Y-%m-%dT%H:%M:%S')\n",
    "    date_to = datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S')\n",
    "    while count == per_page:\n",
    "        path = (\"/vacancies?per_page={}&date_from={}&date_to={}&page={}\"\n",
    "                .format(per_page, date_from, date_to, page))\n",
    "        print path\n",
    "\n",
    "        conn.request(\"GET\", path, headers=headers)\n",
    "        r1 = conn.getresponse()\n",
    "        vacancies = r1.read()\n",
    "        conn.close()\n",
    "\n",
    "        count = len(json.loads(vacancies)['items'])\n",
    "        page = page+1\n",
    "        for item in json.loads(vacancies)['items']:\n",
    "            vacancy_ids.append(item['id'])\n",
    "    return vacancy_ids\n",
    "        \n",
    "\n",
    "def process_vacancies(vacancy_ids):\n",
    "    headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "    for vac_id in vacancy_ids:\n",
    "        conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "        conn.request(\"GET\", \"/vacancies/{}\".format(vac_id), headers=headers)\n",
    "        r1 = conn.getresponse()\n",
    "        if r1 != 200:\n",
    "            conn.close()\n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"/vacancies/{}\".format(vac_id), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "        vacancy_txt = r1.read()\n",
    "        conn.close()\n",
    "        vacancy = json.loads(vacancy_txt)\n",
    "        \n",
    "        feature = []\n",
    "\n",
    "        #description\n",
    "        p_doc = ''\n",
    "        doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "        doc = re.sub('&quot;', '', doc)\n",
    "        doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', doc.strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word.strip())\n",
    "            if len(word.strip()) > 1:\n",
    "                p_doc = p_doc + \" \" + word\n",
    "\n",
    "        #title\n",
    "        p_title = ''\n",
    "        title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', title.strip())\n",
    "        for title_word in words:\n",
    "            title_word = stemmer.stemWord(title_word)\n",
    "            if len(title_word.strip()) > 1:\n",
    "                p_title = p_title + \" \" + title_word.strip()\n",
    "\n",
    "        #keyskills\n",
    "        p_skills = ''\n",
    "        vac_skills = vacancy['key_skills']\n",
    "        for skill in vac_skills:\n",
    "            words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word)\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_skills = p_skills + \" \" + word.strip()\n",
    "                    \n",
    "        #salary\n",
    "        salary = None\n",
    "        if vacancy['salary'] != None:\n",
    "            if vacancy['salary']['from'] == None and vacancy['salary']['to'] != None:\n",
    "                salary = vacancy['salary']['to']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] == None and vacancy['salary']['from'] != None:\n",
    "                salary = vacancy['salary']['from']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] != None and vacancy['salary']['from'] != None:\n",
    "                salary = ((vacancy['salary']['from'] + vacancy['salary']['to'])/2)/currency_rates[vacancy['salary']['currency']]\n",
    "        max_salary = 500000.0\n",
    "        if salary >= max_salary:\n",
    "            salary = max_salary\n",
    "\n",
    "        try:\n",
    "            area_id = areas_map[vacancy['area']['id']]\n",
    "        except KeyError:\n",
    "            print 'missed area id {} for vacancy {}'.format(vacancy['area']['id'], vacancy['id'])\n",
    "            \n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/vacancies/{}\".format(vacancy['id']), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "            if r1 != 200:\n",
    "                conn.close()\n",
    "                conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "                conn.request(\"GET\", \"https://api.hh.ru/vacancies/{}\".format(vacancy['id']), headers=headers)\n",
    "                r1 = conn.getresponse()\n",
    "            missed_area_vacancy = r1.read()\n",
    "            conn.close()\n",
    "            missed_area_vacancy_json = json.loads(missed_area_vacancy)\n",
    "            \n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/areas/{}\"\n",
    "                         .format(missed_area_vacancy_json['area']['id']), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "            missed_area = r1.read()\n",
    "            conn.close()\n",
    "            missed_area_json = json.loads(missed_area)\n",
    "            areas_map[vacancy['area']['id']] = missed_area_json['parent_id']\n",
    "            area_id = missed_area_json['parent_id']\n",
    "\n",
    "        p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "        \n",
    "        #specializations\n",
    "        specializations = set()\n",
    "        try:\n",
    "            if vacancy['specializations'] != None:\n",
    "                for spec in vacancy['specializations']:\n",
    "                    specializations.add(spec['profarea_id'])\n",
    "        except KeyError:\n",
    "            print 'cant find specialization'\n",
    "\n",
    "        feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "        tfidf_feature_p_doc = tfidf_transformer.transform(feature_p_doc)\n",
    "            \n",
    "        data = {}\n",
    "        data['features'] = json.dumps(tfidf_feature_p_doc.toarray()[0].tolist()).encode(\"zlib\")\n",
    "        data['salary'] = salary\n",
    "        data['area'] = area_id\n",
    "        data['specializations'] = specializations\n",
    "        r.hmset(vacancy['id'], data)\n",
    "        r.expire(vacancy['id'], timeout)\n",
    "\n",
    "\n",
    "ids = get_vacancy_ids()\n",
    "print len(ids)\n",
    "vac_id_chunks=[ids[x:x+100] for x in xrange(0, len(ids), 100)]\n",
    "t_num = 1;\n",
    "threads = []\n",
    "for vac_id_chunk in vac_id_chunks:\n",
    "    print 'starting t{}'.format(t_num)\n",
    "    t_num = t_num + 1\n",
    "    t = threading.Thread(target=process_vacancies, kwargs={'vacancy_ids': vac_id_chunk})\n",
    "    threads.append(t)\n",
    "    t.start()\n",
    "    \n",
    "for t in threads:\n",
    "    t.join()\n",
    "    \n",
    "\n",
    "print \"{} sec\".format(time.time() - start_time)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start at 2016-06-22 14:08:44.714534\n",
      "processed 500\n",
      "processed 1000\n",
      "processed 1501\n",
      "processed 1644before 0.00797743695657 after 0.0119661554349\n",
      "before 0.043461698719 after 0.0651925480784\n",
      "before 0.0171287354928 after 0.0256931032392\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0560606400471 after 0.0840909600706\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.00761347359575 after 0.0114202103936\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.00454888075861 after 0.00682332113791\n",
      "before 0.00235002004021 after 0.00352503006031\n",
      "before 0.0203659420272 after 0.0305489130409\n",
      "before 0.0288252018752 after 0.0432378028128\n",
      "before 0.0592338354263 after 0.0888507531394\n",
      "before 0.109405729365 after 0.164108594048\n",
      "before 0.00768819279785 after 0.0115322891968\n",
      "before 0.0609743749251 after 0.0914615623877\n",
      "before 0.0186892957459 after 0.0280339436189\n",
      "before 0.0369376563819 after 0.0554064845729\n",
      "before 0.0130376495532 after 0.0195564743297\n",
      "before 0.0191135353224 after 0.0286703029837\n",
      "before 0.011544124681 after 0.0173161870215\n",
      "before 0.0200216683004 after 0.0300325024506\n",
      "before 0.0277694950151 after 0.0416542425227\n",
      "before 0.139622910346 after 0.209434365519\n",
      "before 0.0185385089646 after 0.0278077634469\n",
      "before 0.00561074443393 after 0.00841611665089\n",
      "before 0.00173315850393 after 0.0025997377559\n",
      "before 0.0563554382835 after 0.0845331574253\n",
      "before 0.180843843426 after 0.271265765139\n",
      "before 0.0102632884429 after 0.0153949326644\n",
      "before 0.0848423205952 after 0.127263480893\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0189324728442 after 0.0283987092663\n",
      "before 0.0130205884561 after 0.0195308826841\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0109634120242 after 0.0164451180363\n",
      "before 0.020829659421 after 0.0312444891314\n",
      "before 0.058958329836 after 0.0884374947541\n",
      "before 0.0106205310945 after 0.0159307966417\n",
      "before 0.00762409823049 after 0.0114361473457\n",
      "before 0.0210983495952 after 0.0316475243928\n",
      "before 0.0039503767252 after 0.00592556508781\n",
      "before 0.00729807896234 after 0.0109471184435\n",
      "before 0.0183360611263 after 0.0275040916895\n",
      "before 0.0225860087985 after 0.0338790131978\n",
      "before 0.0100006722728 after 0.0150010084092\n",
      "before 0.107633808932 after 0.161450713398\n",
      "before 0.017273636514 after 0.025910454771\n",
      "before 0.00520897409685 after 0.00781346114527\n",
      "before 0.0237297773736 after 0.0355946660604\n",
      "before 0.0287410297145 after 0.0431115445717\n",
      "before 0.080640625833 after 0.120960938749\n",
      "before 0.0273066640747 after 0.0409599961121\n",
      "before 0.024407434683 after 0.0366111520245\n",
      "before 0.019934001557 after 0.0299010023354\n",
      "before 0.00705393813528 after 0.0105809072029\n",
      "before 0.0110555145929 after 0.0165832718894\n",
      "before 0.0115958518154 after 0.0173937777231\n",
      "before 0.0269269602431 after 0.0403904403646\n",
      "before 0.0108799983916 after 0.0163199975873\n",
      "before 0.0248059047051 after 0.0372088570577\n",
      "before 0.136120780726 after 0.204181171088\n",
      "before 0.0176571544799 after 0.0264857317198\n",
      "before 0.00660552775509 after 0.00990829163264\n",
      "before 0.122461955607 after 0.18369293341\n",
      "before 0.00867231452441 after 0.0130084717866\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0126817943231 after 0.0190226914846\n",
      "before 0.0109022264198 after 0.0163533396297\n",
      "before 0.0163112466179 after 0.0244668699269\n",
      "before 0.0245717767654 after 0.0368576651481\n",
      "before 0.0112617162862 after 0.0168925744293\n",
      "before 0.00501683998231 after 0.00752525997347\n",
      "before 0.0145123143574 after 0.021768471536\n",
      "before 0.0332195120087 after 0.0498292680131\n",
      "before 0.023281312178 after 0.0349219682669\n",
      "before 0.0219801094956 after 0.0329701642434\n",
      "before 0.00744590133843 after 0.0111688520076\n",
      "before 0.010691611273 after 0.0160374169095\n",
      "before 0.0104300215513 after 0.015645032327\n",
      "before 0.0141855064195 after 0.0212782596293\n",
      "before 0.0113769106794 after 0.0170653660191\n",
      "before 0.0079110666659 after 0.0118665999988\n",
      "before 0.140860482799 after 0.211290724198\n",
      "before 0.0109672031107 after 0.0164508046661\n",
      "before 0.0101263907281 after 0.0151895860922\n",
      "before 0.0191013967298 after 0.0286520950947\n",
      "before 0.0113769106794 after 0.0170653660191\n",
      "before 0.0113622568485 after 0.0170433852727\n",
      "before 0.0899088792015 after 0.134863318802\n",
      "before 0.0832995522803 after 0.12494932842\n",
      "before 0.180843843426 after 0.271265765139\n",
      "before 0.00489138281888 after 0.00733707422832\n",
      "before 0.0233424351848 after 0.0350136527772\n",
      "before 0.0266770193153 after 0.040015528973\n",
      "before 0.010186481322 after 0.0152797219829\n",
      "before 0.193204250318 after 0.289806375476\n",
      "before 0.00957890813173 after 0.0143683621976\n",
      "before 0.0110328496915 after 0.0165492745372\n",
      "before 0.022438791225 after 0.0336581868374\n",
      "before 0.0199885138709 after 0.0299827708063\n",
      "before 0.00950394204176 after 0.0142559130626\n",
      "before 0.079308441614 after 0.118962662421\n",
      "before 0.0309146294108 after 0.0463719441162\n",
      "before 0.0250210897652 after 0.0375316346478\n",
      "before 0.0074546283714 after 0.0111819425571\n",
      "before 0.0274274293093 after 0.0411411439639\n",
      "before 0.00855562911994 after 0.0128334436799\n",
      "before 0.0347083112854 after 0.0520624669281\n",
      "before 0.0102541563655 after 0.0153812345482\n",
      "before 0.0683670579591 after 0.102550586939\n",
      "before 0.0296284530777 after 0.0444426796166\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0601982198712 after 0.0902973298068\n",
      "before 0.0121372309917 after 0.0182058464876\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.00682916441627 after 0.0102437466244\n",
      "before 0.00772045726954 after 0.0115806859043\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0344550183119 after 0.0516825274678before 0.0167789996045 after 0.0251684994068\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "\n",
      "before 0.00537327441607 after 0.0080599116241\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0691788768745 after 0.103768315312\n",
      "before 0.00789258911827 after 0.0118388836774\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.00859713582979 after 0.0128957037447\n",
      "before 0.018587676283 after 0.0278815144245\n",
      "before 0.00146720594522 after 0.00220080891783\n",
      "before 0.0160754707766 after 0.0241132061649before 0.0465774398819 after 0.0698661598229\n",
      "before 0.0666154233224 after 0.0999231349836\n",
      "\n",
      "before 0.00828597204489 after 0.0124289580673before 0.0737635580957 after 0.110645337144\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "\n",
      "before 0.0154142455755 after 0.0231213683633\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0137124678188 after 0.0205687017282\n",
      "before 0.0235519608883 after 0.0353279413324\n",
      "before 0.00865082307223 after 0.0129762346083\n",
      "before 0.0137124678188 after 0.0205687017282\n",
      "before 0.0911185572785 after 0.136677835918\n",
      "before 0.00376338829407 after 0.00564508244111\n",
      "before 0.0207745800092 after 0.0311618700138\n",
      "before 0.0199913178222 after 0.0299869767332\n",
      "before 0.0342900957904 after 0.0514351436856\n",
      "before 0.0216328457035 after 0.0324492685553\n",
      "before 0.0199977860548 after 0.0299966790823\n",
      "before 0.00873572290006 after 0.0131035843501\n",
      "before 0.0612025850796 after 0.0918038776193\n",
      "before 0.0328189754381 after 0.0492284631572\n",
      "before 0.0433658555827 after 0.065048783374\n",
      "before 0.0932207030704 after 0.139831054606\n",
      "before 0.00857187120792 after 0.0128578068119\n",
      "before 0.0113391987946 after 0.0170087981919\n",
      "before 0.0140423261529 after 0.0210634892294\n",
      "before 0.00617132723112 after 0.00925699084668\n",
      "before 0.173375482318 after 0.260063223477\n",
      "before 0.024873316914 after 0.037309975371\n",
      "before 0.20750565036 after 0.31125847554\n",
      "before 0.00216415214974 after 0.00324622822461\n",
      "before 0.00952002285879 after 0.0142800342882\n",
      "before 0.0137124678188 after 0.0205687017282\n",
      "before 0.0101619717629 after 0.0152429576444\n",
      "before 0.00816408694675 after 0.0122461304201\n",
      "before 0.0135159668491 after 0.0202739502736\n",
      "before 0.00508172135009 after 0.00762258202513\n",
      "before 0.0105121698055 after 0.0157682547082\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0187716745481 after 0.0281575118222\n",
      "before 0.0056387118537 after 0.00845806778055\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0107687946765 after 0.0161531920148\n",
      "before 0.0998893774577 after 0.149834066186\n",
      "before 0.066011521233 after 0.0990172818495\n",
      "before 0.0105156878022 after 0.0157735317033\n",
      "before 0.0238311859464 after 0.0357467789196\n",
      "before 0.0242598980976 after 0.0363898471464\n",
      "before 0.0137124678188 after 0.0205687017282\n",
      "before 0.0137124678188 after 0.0205687017282\n",
      "before 0.0213966064862 after 0.0320949097293\n",
      "before 0.00493273769407 after 0.0073991065411\n",
      "before 0.0113900076932 after 0.0170850115398\n",
      "before 0.0314956573311 after 0.0472434859966\n",
      "before 0.0105226536379 after 0.0157839804569\n",
      "before 0.0254434500042 after 0.0381651750063\n",
      "before 0.128651720741 after 0.192977581112\n",
      "before 0.0274052385936 after 0.0411078578905\n",
      "before 0.0069653830752 after 0.0104480746128\n",
      "before 0.013519762864 after 0.0202796442959\n",
      "before 0.00637173067802 after 0.00955759601703\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0261190833085 after 0.0391786249628\n",
      "before 0.0200767815794 after 0.0301151723691\n",
      "before 0.0188803756826 after 0.0283205635238\n",
      "before 0.0275185608448 after 0.0412778412672\n",
      "before 0.0186722915394 after 0.0280084373091\n",
      "before 0.00485887602101 after 0.00728831403151\n",
      "before 0.00605346128997 after 0.00908019193495\n",
      "before 0.0258501852213 after 0.0387752778319\n",
      "before 0.0273768791084 after 0.0410653186626\n",
      "before 0.0115136201667 after 0.0172704302501\n",
      "before 0.0108592622836 after 0.0162888934255\n",
      "before 0.0105005242337 after 0.0157507863506\n",
      "before 0.0293610062164 after 0.0440415093246\n",
      "before 0.0106868617636 after 0.0160302926454\n",
      "before 0.0105860662945 after 0.0158790994418\n",
      "before 0.160990982704 after 0.241486474056\n",
      "before 0.0191695729719 after 0.0287543594579\n",
      "before 0.0130229809774 after 0.0195344714661\n",
      "before 0.152175903945 after 0.228263855917\n",
      "before 0.00909472054498 after 0.0136420808175\n",
      "before 0.0105104411141 after 0.0157656616712\n",
      "before 0.0134383979265 after 0.0201575968898\n",
      "before 0.0136133404359 after 0.0204200106539\n",
      "before 0.0130229809774 after 0.0195344714661\n",
      "before 0.0101975344736 after 0.0152963017104\n",
      "before 0.0110618035812 after 0.0165927053717\n",
      "before 0.0981445075381 after 0.147216761307\n",
      "before 0.0146685466933 after 0.02200282004\n",
      "before 0.037914371719 after 0.0568715575786\n",
      "before 0.0306093854381 after 0.0459140781572\n",
      "before 0.030158443114 after 0.045237664671\n",
      "before 0.00756289844656 after 0.0113443476698\n",
      "before 0.214224211697 after 0.321336317545\n",
      "before 0.0136432448494 after 0.0204648672742\n",
      "before 0.0177130909512 after 0.0265696364268\n",
      "before 0.0272266088603 after 0.0408399132905\n",
      "before 0.026670057132 after 0.040005085698\n",
      "before 0.0135903026385 after 0.0203854539578\n",
      "before 0.0936068868479 after 0.140410330272\n",
      "before 0.0130237098978 after 0.0195355648467\n",
      "before 0.156962014068 after 0.235443021102\n",
      "before 0.00847033490024 after 0.0127055023504\n",
      "before 0.00898896373364 after 0.0134834456005\n",
      "before 0.0213820903308 after 0.0320731354961\n",
      "before 0.0385386444754 after 0.0578079667131\n",
      "before 0.0117167939423 after 0.0175751909134\n",
      "before 0.110965316114 after 0.166447974171\n",
      "before 0.0334815157244 after 0.0502222735866\n",
      "before 0.0111605593946 after 0.0167408390919\n",
      "before 0.117909515738 after 0.176864273607\n",
      "before 0.0128545234209 after 0.0192817851313\n",
      "before 0.20750565036 after 0.31125847554\n",
      "before 0.00484785624788 after 0.00727178437183\n",
      "before 0.00792221236805 after 0.0118833185521\n",
      "before 0.0266124343696 after 0.0399186515544\n",
      "before 0.0100859228632 after 0.0151288842947\n",
      "\n",
      "before 0.0287259465278 after 0.0430889197917before 0.0163469393666 after 0.0245204090499\n",
      "before 0.0108369997189 after 0.0162554995783\n",
      "\n",
      "before 0.0172614450474 after 0.025892167571\n",
      "before 0.0135945642776 after 0.0203918464163\n",
      "before 0.0576760313974 after 0.0865140470961\n",
      "before 0.0207401788125 after 0.0311102682188\n",
      "before 0.00940453552642 after 0.0141068032896\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0350616539364 after 0.0525924809045\n",
      "before 0.0156216079145 after 0.0234324118717\n",
      "before 0.00122324356778 after 0.00183486535167\n",
      "before 0.0310563564538 after 0.0465845346806\n",
      "before 0.00895659438549 after 0.0134348915782\n",
      "before 0.0606801389432 after 0.0910202084148\n",
      "before 0.0106807528361 after 0.0160211292542\n",
      "before 0.0753259208781 after 0.112988881317\n",
      "before 0.0614983862838 after 0.0922475794257\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0743898534053 after 0.111584780108\n",
      "before 0.0128512134332 after 0.0192768201497\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0906000968483 after 0.135900145273\n",
      "before 0.0173202483621 after 0.0259803725431\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.00613049316193 after 0.0091957397429\n",
      "before 0.0180358043435 after 0.0270537065152\n",
      "before 0.0209515217346 after 0.0314272826019\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0510260122472 after 0.0765390183708\n",
      "before 0.0338645770159 after 0.0507968655238\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0815775242338 after 0.122366286351\n",
      "before 0.205425317862 after 0.308137976793\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0117074124455 after 0.0175611186683\n",
      "before 0.00524844719444 after 0.00787267079167\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0207374602206 after 0.0311061903309\n",
      "before 0.00742781724115 after 0.0111417258617\n",
      "before 0.0198843467412 after 0.0298265201118\n",
      "before 0.00793706348114 after 0.0119055952217\n",
      "before 0.0090158800905 after 0.0135238201358\n",
      "before 0.0563380691655 after 0.0845071037482\n",
      "before 0.00680658831633 after 0.0102098824745\n",
      "before 0.0134024921931 after 0.0201037382897before 0.00690820674475 after 0.0103623101171before 0.00876423936246 after 0.0131463590437\n",
      "\n",
      "\n",
      "before 0.00470112462948 after 0.00705168694422\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0832800867953 after 0.124920130193\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.00721239149389 after 0.0108185872408\n",
      "before 0.0198686115037 after 0.0298029172555\n",
      "before 0.0196358151076 after 0.0294537226615\n",
      "before 0.00313762395714 after 0.00470643593571\n",
      "before 0.00949611313468 after 0.014244169702\n",
      "before 0.100240458912 after 0.150360688369\n",
      "before 0.0285884468031 after 0.0428826702046\n",
      "before 0.0212127933786 after 0.0318191900679\n",
      "before 0.0166672245413 after 0.025000836812\n",
      "before 0.0072831744461 after 0.0109247616692\n",
      "before 0.0058072011442 after 0.0087108017163\n",
      "before 0.0166726172567 after 0.025008925885\n",
      "before 0.0361551178793 after 0.054232676819\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0273619397035 after 0.0410429095553\n",
      "before 0.0094537525795 after 0.0141806288693\n",
      "before 0.0157410063571 after 0.0236115095357\n",
      "before 0.00514517840165 after 0.00771776760248\n",
      "before 0.144547153893 after 0.216820730839\n",
      "before 0.00405095743968 after 0.00607643615952\n",
      "before 0.184858018698 after 0.277287028047\n",
      "before 0.00180430375537 after 0.00270645563306\n",
      "before 0.0228247379888 after 0.0342371069832\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.00847227114603 after 0.012708406719\n",
      "before 0.00882584857638 after 0.0132387728646\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0156503797045 after 0.0234755695568\n",
      "before 0.0108575607711 after 0.0162863411566\n",
      "before 0.0599623233616 after 0.0899434850425\n",
      "before 0.00897819345929 after 0.0134672901889\n",
      "before 0.00876279811248 after 0.0131441971687\n",
      "before 0.0202260387504 after 0.0303390581256\n",
      "before 0.00876717239779 after 0.0131507585967\n",
      "before 0.0108575607711 after 0.0162863411566\n",
      "before 0.0178388462382 after 0.0267582693572\n",
      "before 0.00411253762668 after 0.00616880644002\n",
      "before 0.0818253483432 after 0.122738022515\n",
      "before 0.112052120577 after 0.168078180865\n",
      "before 0.0228483819482 after 0.0342725729224\n",
      "before 0.0226145267757 after 0.0339217901636\n",
      "before 0.0217760845078 after 0.0326641267617\n",
      "before 0.0167384776546 after 0.0251077164819\n",
      "before 0.197761800588 after 0.296642700882\n",
      "before 0.0229428613329 after 0.0344142919993\n",
      "before 0.0299208339742 after 0.0448812509613\n",
      "before 0.0226994542036 after 0.0340491813054\n",
      "before 0.00504691083748 after 0.00757036625623\n",
      "before 0.0215518979464 after 0.0323278469197\n",
      "before 0.0821050328178 after 0.123157549227\n",
      "before 0.0280322978984 after 0.0420484468475\n",
      "before 0.00905361839799 after 0.013580427597\n",
      "before 0.00706191432594 after 0.0105928714889\n",
      "before 0.140036981935 after 0.210055472902\n",
      "before 0.0089098840927 after 0.013364826139\n",
      "before 0.0321305602114 after 0.0481958403172\n",
      "before 0.126872574554 after 0.190308861832\n",
      "before 0.0159821168333 after 0.0239731752499\n",
      "before 0.0279143148804 after 0.0418714723207\n",
      "before 0.0122295070094 after 0.0183442605141\n",
      "before 0.0075824790948 after 0.0113737186422\n",
      "before 0.0107171138058 after 0.0160756707087\n",
      "before 0.0251437923037 after 0.0377156884556\n",
      "before 0.0113497571206 after 0.017024635681\n",
      "before 0.00660493187978 after 0.00990739781967\n",
      "before 0.0113746891226 after 0.0170620336839\n",
      "before 0.00922248176732 after 0.013833722651\n",
      "before 0.0342874679119 after 0.0514312018679before 0.0222354441415 after 0.0333531662123\n",
      "before 0.0316100895602 after 0.0474151343403\n",
      "\n",
      "before 0.0108581684889 after 0.0162872527334\n",
      "before 0.00630536354399 after 0.00945804531598\n",
      "before 0.0688424687386 after 0.103263703108\n",
      "before 0.00749430719252 after 0.0112414607888\n",
      "before 0.0147678140496 after 0.0221517210744\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.00976856239682 after 0.0146528435952\n",
      "before 0.140273127843 after 0.210409691764\n",
      "before 0.00146007110812 after 0.00219010666218\n",
      "before 0.00930481677552 after 0.0139572251633\n",
      "before 0.0178267437833 after 0.0267401156749\n",
      "before 0.0662914809371 after 0.0994372214056\n",
      "before 0.184858018698 after 0.277287028047\n",
      "before 0.0925502532092 after 0.138825379814\n",
      "before 0.0734048552343 after 0.110107282851\n",
      "before 0.0221874027103 after 0.0332811040655\n",
      "before 0.00404176999968 after 0.00606265499952\n",
      "before 0.0153392880472 after 0.0230089320708\n",
      "before 0.0084771953993 after 0.0127157930989\n",
      "before 0.00840886741747 after 0.0126133011262\n",
      "before 0.0136457858206 after 0.020468678731\n",
      "before 0.0172915672879 after 0.0259373509319\n",
      "before 0.0090350575823 after 0.0135525863735\n",
      "before 0.0136457858206 after 0.020468678731\n",
      "before 0.0292317126938 after 0.0438475690407\n",
      "before 0.00784077901819 after 0.0117611685273\n",
      "before 0.0206735558519 after 0.0310103337779\n",
      "before 0.0258924034451 after 0.0388386051677\n",
      "before 0.0130240962164 after 0.0195361443245\n",
      "before 0.0215276479086 after 0.0322914718629\n",
      "before 0.00890479093845 after 0.0133571864077\n",
      "before 0.00746732011749 after 0.0112009801762\n",
      "before 0.0609049646426 after 0.0913574469639\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0711734899835 after 0.106760234975\n",
      "before 0.0927673825718 after 0.139151073858\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0620205431844 after 0.0930308147767\n",
      "before 0.0139740400954 after 0.0209610601431\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.00511113409271 after 0.00766670113907\n",
      "before 0.0247523611171 after 0.0371285416757\n",
      "before 0.0174677687754 after 0.0262016531632\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.00947372819069 after 0.014210592286\n",
      "before 0.00437575194699 after 0.00656362792048\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.00812438602363 after 0.0121865790354\n",
      "before 0.00619274321541 after 0.00928911482311\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0104610504521 after 0.0156915756782\n",
      "before 0.00751674798245 after 0.0112751219737\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.00561129151053 after 0.00841693726579\n",
      "before 0.0159972977937 after 0.0239959466906before 0.0165780402744 after 0.0248670604116\n",
      "before 0.0994036280383 after 0.149105442057\n",
      "\n",
      "before 0.0469703526981 after 0.0704555290471\n",
      "before 0.0237152979007 after 0.0355729468511\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.00824567840995 after 0.0123685176149before 0.0136457858206 after 0.020468678731\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "\n",
      "before 0.0113346195252 after 0.0170019292878\n",
      "before 0.0234374306788 after 0.0351561460182\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0253197217222 after 0.0379795825833\n",
      "before 0.0906754592493 after 0.136013188874\n",
      "before 0.00860875521284 after 0.0129131328193\n",
      "before 0.00693151129754 after 0.0103972669463\n",
      "before 0.0198941025699 after 0.0298411538549\n",
      "before 0.00374508741238 after 0.00561763111857\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0199005393484 after 0.0298508090226\n",
      "before 0.0341233473878 after 0.0511850210818\n",
      "before 0.0187885628016 after 0.0281828442023\n",
      "before 0.0326593809079 after 0.0489890713619\n",
      "before 0.00869324218355 after 0.0130398632753\n",
      "before 0.00483524792094 after 0.00725287188142\n",
      "before 0.00853018728148 after 0.0127952809222\n",
      "before 0.0431549729072 after 0.0647324593608\n",
      "before 0.027243748805 after 0.0408656232074\n",
      "before 0.00614131684668 after 0.00921197527001\n",
      "before 0.0112840577038 after 0.0169260865558\n",
      "before 0.0104494615114 after 0.0156741922671\n",
      "before 0.206496576605 after 0.309744864908\n",
      "before 0.172532379257 after 0.258798568886\n",
      "before 0.0105345875919 after 0.0158018813879\n",
      "before 0.0136457858206 after 0.020468678731\n",
      "before 0.00215362815132 after 0.00323044222698\n",
      "before 0.0129596518666 after 0.0194394778\n",
      "before 0.0134502404103 after 0.0201753606155\n",
      "before 0.0101125553784 after 0.0151688330677\n",
      "before 0.0104593301672 after 0.0156889952508\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.00505700958135 after 0.00758551437202\n",
      "before 0.0129596518666 after 0.0194394778\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.018680390267 after 0.0280205854005\n",
      "before 0.0976672432006 after 0.146500864801\n",
      "before 0.0656905155472 after 0.0985357733209\n",
      "before 0.0107164273889 after 0.0160746410833\n",
      "before 0.0269928393975 after 0.0404892590962\n",
      "before 0.0241419252789 after 0.0362128879184\n",
      "before 0.0104645513413 after 0.0156968270119\n",
      "before 0.213182466427 after 0.31977369964\n",
      "before 0.0212925574928 after 0.0319388362391\n",
      "before 0.0136457858206 after 0.020468678731\n",
      "before 0.0270942093021 after 0.0406413139532\n",
      "before 0.0313424979296 after 0.0470137468944\n",
      "before 0.00490875041402 after 0.00736312562103\n",
      "before 0.0931516883864 after 0.13972753258\n",
      "before 0.128026103681 after 0.192039155522\n",
      "before 0.010471483303 after 0.0157072249546\n",
      "before 0.00842914473203 after 0.012643717098\n",
      "before 0.0134540179657 after 0.0201810269485\n",
      "before 0.0272719703816 after 0.0409079555724\n",
      "before 0.0383512359175 after 0.0575268538763\n",
      "before 0.0259920695071 after 0.0389881042607\n",
      "before 0.00634074575694 after 0.00951111863541\n",
      "before 0.0333186993445 after 0.0499780490168\n",
      "before 0.027384741561 after 0.0410771123415\n",
      "before 0.0199791507277 after 0.0299687260916\n",
      "before 0.0127920134979 after 0.0191880202469\n",
      "before 0.0060240240727 after 0.00903603610905\n",
      "before 0.0185814905453 after 0.0278722358179\n",
      "before 0.00788368765044 after 0.0118255314757\n",
      "before 0.0114576308868 after 0.0171864463303\n",
      "before 0.0257244790373 after 0.0385867185559\n",
      "before 0.00424485080862 after 0.00636727621294before 0.0292182273536 after 0.0438273410303\n",
      "before 0.0108064550635 after 0.0162096825952\n",
      "\n",
      "before 0.1602081039 after 0.24031215585\n",
      "before 0.0106348929053 after 0.015952339358\n",
      "before 0.155012050143 after 0.232518075215\n",
      "before 0.151435891755 after 0.227153837632\n",
      "before 0.019076353761 after 0.0286145306416\n",
      "before 0.0774235964684 after 0.116135394703\n",
      "before 0.0133730486956 after 0.0200595730435\n",
      "before 0.00905049406829 after 0.0135757411024\n",
      "before 0.00830325279006 after 0.0124548791851\n",
      "before 0.0101479451521 after 0.0152219177282\n",
      "before 0.0135471404817 after 0.0203207107226\n",
      "before 0.0782605704076 after 0.117390855611\n",
      "before 0.0145972154044 after 0.0218958231067\n",
      "before 0.0110080114282 after 0.0165120171424\n",
      "before 0.00242342610383 after 0.00363513915575\n",
      "before 0.0300117864163 after 0.0450176796245\n",
      "before 0.0377299989207 after 0.056594998381\n",
      "before 0.0492604538082 after 0.0738906807123\n",
      "before 0.0135768994739 after 0.0203653492108\n",
      "before 0.00752612102715 after 0.0112891815407\n",
      "before 0.0708928587148 after 0.106339288072\n",
      "before 0.0265403640145 after 0.0398105460218\n",
      "before 0.0176269544283 after 0.0264404316425\n",
      "before 0.00708661147794 after 0.0106299172169\n",
      "before 0.0129603772424 after 0.0194405658636\n",
      "before 0.0135242147142 after 0.0202863220713\n",
      "before 0.0254317524368 after 0.0381476286552\n",
      "before 0.00894525153896 after 0.0134178773084\n",
      "before 0.156198727629 after 0.234298091443\n",
      "before 0.0 after 0.0\n",
      "before 0.011659816654 after 0.017489724981\n",
      "before 0.0212781119276 after 0.0319171678914\n",
      "before 0.0121364614324 after 0.0182046921485\n",
      "before 0.0111062870046 after 0.0166594305069\n",
      "before 0.110425705805 after 0.165638558707\n",
      "before 0.00553588777031 after 0.00830383165547\n",
      "before 0.206496576605 after 0.309744864908\n",
      "before 0.110468556156 after 0.165702834235\n",
      "before 0.0558868778394 after 0.083830316759\n",
      "before 0.0264830214643 after 0.0397245321965\n",
      "before 0.00482428173558 after 0.00723642260337\n",
      "before 0.00338926753102 after 0.00508390129653\n",
      "before 0.0101184329977 after 0.0151776494965\n",
      "before 0.0100368762949 after 0.0150553144423\n",
      "before 0.0729934619253 after 0.109490192888\n",
      "before 0.0171775048214 after 0.0257662572322\n",
      "before 0.0107843007588 after 0.0161764511381\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0206393219439 after 0.0309589829159\n",
      "before 0.0135284556294 after 0.0202926834442\n",
      "before 0.0166432510588 after 0.0249648765882\n",
      "before 0.0348911535441 after 0.0523367303161\n",
      "before 0.00935880245862 after 0.0140382036879\n",
      "before 0.0126388323111 after 0.0189582484666\n",
      "before 0.030905333317 after 0.0463579999754\n",
      "before 0.0155456420093 after 0.023318463014\n",
      "before 0.0189157560399 after 0.0283736340599\n",
      "before 0.0106288136848 after 0.0159432205272\n",
      "before 0.00891303959885 after 0.0133695593983\n",
      "before 0.0189157560399 after 0.0283736340599\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0749596204439 after 0.112439430666\n",
      "before 0.00417092249599 after 0.00625638374398\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0740281049489 after 0.111042157423\n",
      "before 0.0226942680556 after 0.0340414020834\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0901595200266 after 0.13523928004\n",
      "before 0.0628541305104 after 0.0942811957657\n",
      "before 0.0208496371321 after 0.0312744556982\n",
      "before 0.00610068134841 after 0.00915102202261\n",
      "before 0.0267260001527 after 0.0400890002291\n",
      "before 0.0336998978574 after 0.0505498467861\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0107196437164 after 0.0160794655747\n",
      "before 0.204426360501 after 0.306639540752\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0458676938918 after 0.0688015408377\n",
      "before 0.00522292465899 after 0.00783438698849\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0196687226055 after 0.0295030839083\n",
      "before 0.00739169670457 after 0.0110875450569\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.00551827490676 after 0.00827741236014\n",
      "before 0.00897203700498 after 0.0134580555075\n",
      "before 0.0197876516758 after 0.0296814775137\n",
      "before 0.018530484726 after 0.0277957270891before 0.00424485080862 after 0.00636727621294before 0.0560641042546 after 0.084096156382\n",
      "\n",
      "\n",
      "before 0.00424485080862 after 0.00636727621294before 0.0268324840449 after 0.0402487260673\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "\n",
      "before 0.0 after 0.0\n",
      "before 0.0230818658431 after 0.0346227987646\n",
      "before 0.0109480607651 after 0.0164220911476\n",
      "before 0.0636579447488 after 0.0954869171232\n",
      "before 0.0651926921693 after 0.097789038254\n",
      "before 0.00612898424559 after 0.00919347636838\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.051559352924 after 0.0773390293861\n",
      "before 0.0136664375881 after 0.0204996563822\n",
      "before 0.0091159861184 after 0.0136739791776\n",
      "before 0.0299300359726 after 0.0448950539589\n",
      "before 0.0110555058563 after 0.0165832587844\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0187973134378 after 0.0281959701567\n",
      "before 0.0374185331793 after 0.056127799769\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0671061360583 after 0.100659204087\n",
      "before 0.0968082445809 after 0.145212366871\n",
      "before 0.148549993514 after 0.222824990271\n",
      "before 0.0 after 0.0\n",
      "before 0.0265539548427 after 0.0398309322641\n",
      "before 0.00333374232771 after 0.00500061349157\n",
      "before 0.0405184446283 after 0.0607776669424\n",
      "before 0.0 after 0.0\n",
      "before 0.00239696133383 after 0.00359544200074\n",
      "before 0.00280633673118 after 0.00420950509677\n",
      "before 0.00234260831763 after 0.00351391247644\n",
      "before 0.0783175793029 after 0.117476368954\n",
      "before 0.0269013454667 after 0.0403520182001\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0395893898732 after 0.0593840848098\n",
      "before 0.0288081994236 after 0.0432122991354\n",
      "before 0.0061613989944 after 0.0092420984916\n",
      "before 0.0129568726106 after 0.0194353089159\n",
      "before 0.0133824200693 after 0.0200736301039\n",
      "before 0.0152094051063 after 0.0228141076595\n",
      "before 0.0273072203069 after 0.0409608304604\n",
      "before 0.0103565186985 after 0.0155347780477\n",
      "before 0.00987894116358 after 0.0148184117454\n",
      "before 0.0693117801779 after 0.103967670267\n",
      "before 0.0753571799004 after 0.113035769851\n",
      "before 0.0185198602219 after 0.0277797903328\n",
      "before 0.0298622052135 after 0.0447933078202\n",
      "before 0.00285075640617 after 0.00427613460926\n",
      "before 0.00656131762357 after 0.00984197643536\n",
      "before 0.026179681599 after 0.0392695223985\n",
      "before 0.00401899850573 after 0.0060284977586\n",
      "before 0.01855730086 after 0.0278359512901\n",
      "before 0.0205117622501 after 0.0307676433752\n",
      "before 0.0592722606281 after 0.0889083909421\n",
      "before 0.0546391177553 after 0.081958676633\n",
      "before 0.0113759951336 after 0.0170639927003\n",
      "before 0.0842614525422 after 0.126392178813\n",
      "before 0.0721359729108 after 0.108203959366\n",
      "before 0.0299300359726 after 0.0448950539589\n",
      "before 0.0891398390246 after 0.133709758537\n",
      "before 0.00699025617879 after 0.0104853842682\n",
      "before 0.00611993630656 after 0.00917990445983\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0197211427095 after 0.0295817140642\n",
      "before 0.0165678291619 after 0.0248517437428\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0061613989944 after 0.0092420984916\n",
      "before 0.0781309765832 after 0.117196464875\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0185606334765 after 0.0278409502147\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.00414624441679 after 0.00621936662518\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0632271135061 after 0.0948406702591\n",
      "before 0.00081510299222 after 0.00122265448833\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0361528423516 after 0.0542292635274before 0.0111960859605 after 0.0167941289407\n",
      "before 0.00652436122529 after 0.00978654183794\n",
      "\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.0222889167421 after 0.0334333751131before 0.0140210543717 after 0.0210315815576\n",
      "before 0.0175439261589 after 0.0263158892383\n",
      "\n",
      "before 0.013274889623 after 0.0199123344345\n",
      "before 0.0175439261589 after 0.0263158892383\n",
      "before 0.0190938989702 after 0.0286408484553\n",
      "before 0.0184624231563 after 0.0276936347345\n",
      "before 0.0180964436649 after 0.0271446654974\n",
      "before 0.013274889623 after 0.0199123344345\n",
      "before 0.0135981114639 after 0.0203971671958\n",
      "before 0.0441238377704 after 0.0661857566555\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.017389503608 after 0.026084255412\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0195089338666 after 0.0292634007998\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.00561649370189 after 0.00842474055284\n",
      "before 0.0443531387432 after 0.0665297081149\n",
      "before 0.0274492491242 after 0.0411738736863\n",
      "before 0.070590182939 after 0.105885274408\n",
      "before 0.0173129143806 after 0.0259693715709\n",
      "before 0.0288159955812 after 0.0432239933719\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.0316112861208 after 0.0474169291812\n",
      "before 0.0386215893237 after 0.0579323839855\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.0588236920867 after 0.08823553813\n",
      "before 0.0277186381182 after 0.0415779571773\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0305021226652 after 0.0457531839979\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.00430885259833 after 0.00646327889749\n",
      "before 0.0485125970366 after 0.0727688955549\n",
      "before 0.0111081764371 after 0.0166622646557\n",
      "before 0.0061613989944 after 0.0092420984916\n",
      "before 0.018598673988 after 0.027898010982\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0292898376775 after 0.0439347565162\n",
      "before 0.0176958382483 after 0.0265437573725\n",
      "before 0.0105792849845 after 0.0158689274768\n",
      "before 0.134813723049 after 0.202220584574\n",
      "before 0.0293675091824 after 0.0440512637736\n",
      "before 0.0728371851736 after 0.10925577776\n",
      "before 0.0184329296339 after 0.0276493944508\n",
      "before 0.0 after 0.0\n",
      "before 0.0187102329006 after 0.0280653493509\n",
      "before 0.00976418641394 after 0.0146462796209\n",
      "before 0.0105100669064 after 0.0157651003595\n",
      "before 0.00414841772164 after 0.00622262658245\n",
      "before 0.00818735091561 after 0.0122810263734\n",
      "before 0.0185558005224 after 0.0278337007835\n",
      "before 0.0185430980461 after 0.0278146470692\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0711280517267 after 0.10669207759\n",
      "before 0.00615650219043 after 0.00923475328565\n",
      "before 0.011974246178 after 0.017961369267\n",
      "before 0.0463510339618 after 0.0695265509427\n",
      "before 0.0169800180238 after 0.0254700270357\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0158787197005 after 0.0238180795507\n",
      "before 0.02656638106 after 0.03984957159\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0261428237038 after 0.0392142355558\n",
      "before 0.0283750427677 after 0.0425625641516\n",
      "before 0.0 after 0.0\n",
      "before 0.0225170743945 after 0.0337756115918\n",
      "before 0.0014083655035 after 0.00211254825525\n",
      "before 0.018598673988 after 0.027898010982\n",
      "before 0.0355334496626 after 0.0533001744939\n",
      "before 0.0295545303698 after 0.0443317955546\n",
      "before 0.0184725356439 after 0.0277088034659\n",
      "before 0.0272245643226 after 0.0408368464839\n",
      "before 0.00439692023152 after 0.00659538034729\n",
      "before 0.0185972394867 after 0.02789585923\n",
      "before 0.0221068328076 after 0.0331602492115\n",
      "before 0.02459177418 after 0.03688766127\n",
      "before 0.0325662516826 after 0.0488493775239\n",
      "before 0.0184580185423 after 0.0276870278134\n",
      "before 0.0683183684419 after 0.102477552663\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.0184681820819 after 0.0277022731228\n",
      "before 0.0310833543078 after 0.0466250314616\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.0185972394867 after 0.02789585923\n",
      "before 0.012351405568 after 0.018527108352\n",
      "before 0.136051344433 after 0.204077016649\n",
      "before 0.0168056713862 after 0.0252085070793\n",
      "before 0.0175439261589 after 0.0263158892383\n",
      "before 0.0175439261589 after 0.0263158892383\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0184527730685 after 0.0276791596028\n",
      "before 0.0172925200263 after 0.0259387800395\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0177751300984 after 0.0266626951477\n",
      "before 0.0061613989944 after 0.0092420984916\n",
      "before 0.0105851069591 after 0.0158776604386\n",
      "before 0.0185256373876 after 0.0277884560813\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0164314232477 after 0.0246471348716\n",
      "before 0.0187102329006 after 0.0280653493509\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0187102329006 after 0.0280653493509\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0187102329006 after 0.0280653493509\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0115875529708 after 0.0173813294562\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0129509824919 after 0.0194264737379\n",
      "before 0.019814811511 after 0.0297222172666\n",
      "before 0.0130256514829 after 0.0195384772244\n",
      "before 0.0533388409605 after 0.0800082614407\n",
      "before 0.0175439261589 after 0.0263158892383\n",
      "before 0.10222774815 after 0.153341622225\n",
      "before 0.049465859361 after 0.0741987890415\n",
      "before 0.0184997883548 after 0.0277496825322\n",
      "before 0.0121972606511 after 0.0182958909767\n",
      "before 0.0171996033619 after 0.0257994050429\n",
      "before 0.0322873299079 after 0.0484309948618\n",
      "before 0.0048868584459 after 0.00733028766885\n",
      "before 0.0242748408071 after 0.0364122612107\n",
      "before 0.0150791040519 after 0.0226186560778\n",
      "before 0.0362266999023 after 0.0543400498534\n",
      "before 0.0614333439097 after 0.0921500158646\n",
      "before 0.00498407863897 after 0.00747611795845\n",
      "before 0.0271872726223 after 0.0407809089335\n",
      "before 0.140276581659 after 0.210414872488\n",
      "before 0.0266322447416 after 0.0399483671123\n",
      "before 0.00636967381243 after 0.00955451071864\n",
      "before 0.0172973767017 after 0.0259460650525\n",
      "before 0.0209054937209 after 0.0313582405814\n",
      "before 0.0223877836528 after 0.0335816754791\n",
      "before 0.0158642669524 after 0.0237964004287\n",
      "before 0.0514407779086 after 0.0771611668629\n",
      "before 0.0186408138096 after 0.0279612207144\n",
      "before 0.00265446891099 after 0.00398170336649\n",
      "before 0.0183290591624 after 0.0274935887436\n",
      "before 0.0186408138096 after 0.0279612207144\n",
      "before 0.0309984050996 after 0.0464976076494\n",
      "before 0.0 after 0.0\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0159791502101 after 0.0239687253152\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.0132167329159 after 0.0198250993739\n",
      "before 0.0178337349729 after 0.0267506024594\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.0300537645647 after 0.045080646847\n",
      "before 0.0846262743921 after 0.126939411588\n",
      "before 0.0064058525207 after 0.00960877878104\n",
      "before 0.0313452126589 after 0.0470178189883\n",
      "before 0.0472168811171 after 0.0708253216757\n",
      "before 0.0187102329006 after 0.0280653493509\n",
      "before 0.0215358637397 after 0.0323037956096\n",
      "before 0.00169342564882 after 0.00254013847323\n",
      "before 0.00356142704235 after 0.00534214056353\n",
      "before 0.0281677968085 after 0.0422516952127\n",
      "before 0.0617847542005 after 0.0926771313007\n",
      "before 0.0141666080191 after 0.0212499120287\n",
      "before 0.0176951631887 after 0.026542744783\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0701213928978 after 0.105182089347\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.08601522616 after 0.12902283924\n",
      "before 0.0453407341689 after 0.0680111012534\n",
      "before 0.0202912800509 after 0.0304369200764\n",
      "before 0.0848332669553 after 0.127249900433\n",
      "before 0.0111867187777 after 0.0167800781666\n",
      "before 0.0116372368124 after 0.0174558552186\n",
      "before 0.0174960554219 after 0.0262440831328\n",
      "before 0.00512641230336 after 0.00768961845503\n",
      "before 0.0106851710634 after 0.0160277565951\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0307872224481 after 0.0461808336721\n",
      "before 0.0165413940663 after 0.0248120910994\n",
      "before 0.00647289166712 after 0.00970933750068\n",
      "before 0.0169468542133 after 0.0254202813199\n",
      "before 0.0272294611666 after 0.0408441917499\n",
      "before 0.0496207550071 after 0.0744311325106\n",
      "before 0.0466177038723 after 0.0699265558085\n",
      "before 0.018439285209 after 0.0276589278136\n",
      "before 0.00809113238855 after 0.0121366985828\n",
      "before 0.0180748230157 after 0.0271122345236\n",
      "before 0.00811480165872 after 0.0121722024881\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0176522072999 after 0.0264783109499\n",
      "before 0.00935798342048 after 0.0140369751307\n",
      "before 0.0203731414586 after 0.0305597121878\n",
      "before 0.0313623801009 after 0.0470435701513\n",
      "before 0.00613236649069 after 0.00919854973603\n",
      "before 0.00664827697377 after 0.00997241546065\n",
      "before 0.016677807716 after 0.025016711574\n",
      "before 0.0188697254416 after 0.0283045881625\n",
      "before 0.0219041529332 after 0.0328562293998\n",
      "before 0.0186942192604 after 0.0280413288905\n",
      "before 0.0427641717877 after 0.0641462576815\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0067159046546 after 0.0100738569819\n",
      "before 0.00613236649069 after 0.00919854973603\n",
      "before 0.0175199786989 after 0.0262799680484\n",
      "before 0.00390177449809 after 0.00585266174714\n",
      "before 0.0285278116016 after 0.0427917174024\n",
      "before 0.00980218295662 after 0.0147032744349\n",
      "before 0.0350130224552 after 0.0525195336827\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.00648022560122 after 0.00972033840183\n",
      "before 0.0036354765098 after 0.0054532147647\n",
      "before 0.0302028318654 after 0.0453042477981\n",
      "before 0.0185558005224 after 0.0278337007835\n",
      "before 0.0440865695107 after 0.066129854266\n",
      "before 0.0318324978012 after 0.0477487467018\n",
      "before 0.00776970784528 after 0.0116545617679\n",
      "before 0.0084522736685 after 0.0126784105028\n",
      "before 0.139129744452 after 0.208694616677\n",
      "before 0.0 after 0.0\n",
      "before 0.0155659583941 after 0.0233489375911\n",
      "before 0.0064058525207 after 0.00960877878104\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0249718220491 after 0.0374577330737\n",
      "before 0.0186249776007 after 0.027937466401\n",
      "before 0.0186249776007 after 0.027937466401\n",
      "before 0.013384943024 after 0.020077414536\n",
      "before 0.018462849836 after 0.027694274754\n",
      "before 0.0259621539284 after 0.0389432308926\n",
      "before 0.0102317587896 after 0.0153476381844\n",
      "before 0.0185972394867 after 0.02789585923\n",
      "before 0.0125023277784 after 0.0187534916676\n",
      "before 0.0172629381011 after 0.0258944071516\n",
      "before 0.0193734402078 after 0.0290601603118\n",
      "before 0.0178373581156 after 0.0267560371734\n",
      "before 0.0135977362347 after 0.0203966043521\n",
      "before 0.0185558005224 after 0.0278337007835\n",
      "before 0.0316884438939 after 0.0475326658408\n",
      "before 0.0185053968411 after 0.0277580952616\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0185558005224 after 0.0278337007835\n",
      "before 0.0161848457128 after 0.0242772685692\n",
      "before 0.0185558005224 after 0.0278337007835\n",
      "before 0.00320488118932 after 0.00480732178397\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.160691087972 after 0.241036631958\n",
      "before 0.0 after 0.0\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0185972394867 after 0.02789585923\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0182095887576 after 0.0273143831363\n",
      "before 0.0285221767725 after 0.0427832651587\n",
      "before 0.0114174973125 after 0.0171262459688\n",
      "before 0.0071545055484 after 0.0107317583226\n",
      "before 0.0345818556729 after 0.0518727835093\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0764857546173 after 0.114728631926\n",
      "before 0.0235066143336 after 0.0352599215004\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.033468837915 after 0.0502032568725\n",
      "before 0.00791971198838 after 0.0118795679826\n",
      "before 0.0256957316893 after 0.0385435975339\n",
      "before 0.0268765562422 after 0.0403148343633\n",
      "before 0.0118496502629 after 0.0177744753944\n",
      "before 0.0211036464774 after 0.0316554697161\n",
      "before 0.0 after 0.0\n",
      "before 0.0174170554185 after 0.0261255831278\n",
      "before 0.0173876262915 after 0.0260814394373\n",
      "before 0.0186408138096 after 0.0279612207144\n",
      "before 0.136051344433 after 0.204077016649\n",
      "before 0.0476946046675 after 0.0715419070013\n",
      "before 0.0186408138096 after 0.0279612207144\n",
      "before 0.0285118289763 after 0.0427677434644\n",
      "before 0.088349122406 after 0.132523683609\n",
      "before 0.0186408138096 after 0.0279612207144\n",
      "before 0.0178795350537 after 0.0268193025806\n",
      "before 0.0351395603589 after 0.0527093405384\n",
      "before 0.0184686425611 after 0.0277029638416\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0186408138096 after 0.0279612207144\n",
      "before 0.0269110639481 after 0.0403665959221\n",
      "before 0.0186249776007 after 0.027937466401\n",
      "before 0.0081771074759 after 0.0122656612138\n",
      "before 0.0194721080477 after 0.0292081620715\n",
      "before 0.00807771825305 after 0.0121165773796\n",
      "before 0.00330075638207 after 0.0049511345731\n",
      "before 0.0227552973995 after 0.0341329460993\n",
      "before 0.0440507718976 after 0.0660761578463\n",
      "before 0.0691668574707 after 0.103750286206\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0134268045744 after 0.0201402068615\n",
      "before 0.018598673988 after 0.027898010982\n",
      "before 0.035068264173 after 0.0526023962595\n",
      "before 0.0028322063096 after 0.0042483094644\n",
      "before 0.00893753762819 after 0.0134063064423\n",
      "before 0.0372698053974 after 0.0559047080961\n",
      "before 0.0124093833168 after 0.0186140749752\n",
      "before 0.0132918550845 after 0.0199377826268\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.04522035404 after 0.06783053106\n",
      "before 0.0063608559518 after 0.0095412839277\n",
      "before 0.0301713540936 after 0.0452570311405\n",
      "before 0.021790699653 after 0.0326860494794\n",
      "before 0.0185010125759 after 0.0277515188639\n",
      "before 0.0570375128714 after 0.0855562693071\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0186408138096 after 0.0279612207144\n",
      "before 0.0506011271823 after 0.0759016907734\n",
      "before 0.000803131744998 after 0.0012046976175\n",
      "before 0.0833015076517 after 0.124952261478\n",
      "before 0.0302660973604 after 0.0453991460406\n",
      "before 0.0295244101459 after 0.0442866152188\n",
      "before 0.0186249776007 after 0.027937466401\n",
      "before 0.0188235018384 after 0.0282352527576\n",
      "before 0.0187102329006 after 0.0280653493509\n",
      "before 0.0186249776007 after 0.027937466401\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.037035866984 after 0.055553800476\n",
      "before 0.01783895035 after 0.0267584255249\n",
      "before 0.0319223117757 after 0.0478834676636before 0.0185558005224 after 0.0278337007835\n",
      "before 0.0185558005224 after 0.0278337007835\n",
      "\n",
      "before 0.0452427473911 after 0.0678641210866\n",
      "before 0.0157598885138 after 0.0236398327707\n",
      "before 0.0 after 0.0\n",
      "before 0.0129690189466 after 0.0194535284199\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0993268627945 after 0.148990294192\n",
      "before 0.0183251200623 after 0.0274876800934\n",
      "before 0.0169121114813 after 0.0253681672219\n",
      "before 0.0 after 0.0\n",
      "before 0.0115733057081 after 0.0173599585621\n",
      "before 0.0143746469825 after 0.0215619704738\n",
      "before 0.0341530937543 after 0.0512296406314\n",
      "before 0.0144729654842 after 0.0217094482263\n",
      "before 0.017393078693 after 0.0260896180395\n",
      "before 0.0187866579252 after 0.0281799868878before 0.018530484726 after 0.0277957270891\n",
      "before 0.009493888817 after 0.0142408332255\n",
      "\n",
      "before 0.010513810142 after 0.015770715213\n",
      "before 0.0 after 0.0\n",
      "before 0.0238878988598 after 0.0358318482897\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.00599941969199 after 0.00899912953799\n",
      "before 0.0 after 0.0\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0185972394867 after 0.02789585923\n",
      "before 0.0355450852609 after 0.0533176278914\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0185972394867 after 0.02789585923\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0766895503936 after 0.11503432559\n",
      "before 0.00924200933701 after 0.0138630140055\n",
      "before 0.0090025283264 after 0.0135037924896\n",
      "before 0.0287164545144 after 0.0430746817717\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0261289550912 after 0.0391934326368\n",
      "before 0.0175355483325 after 0.0263033224988\n",
      "before 0.0379358226887 after 0.056903734033\n",
      "before 0.0185290659439 after 0.0277935989159\n",
      "before 0.0102308084857 after 0.0153462127286\n",
      "before 0.09814656239 after 0.147219843585\n",
      "before 0.018598673988 after 0.027898010982\n",
      "before 0.016123808678 after 0.024185713017\n",
      "before 0.0269210478607 after 0.0403815717911\n",
      "before 0.0501790674362 after 0.0752686011543\n",
      "before 0.0558454724666 after 0.0837682086999\n",
      "before 0.0 after 0.0\n",
      "before 0.0153171007253 after 0.022975651088\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.00237499351834 after 0.00356249027751\n",
      "before 0.00834623374155 after 0.0125193506123\n",
      "before 0.0187102329006 after 0.0280653493509\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.00820700968107 after 0.0123105145216\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0062465767604 after 0.00936986514059\n",
      "before 0.0100752183598 after 0.0151128275397\n",
      "before 0.0494753997335 after 0.0742130996003\n",
      "before 0.00686987299925 after 0.0103048094989\n",
      "before 0.026957651699 after 0.0404364775485\n",
      "before 0.0895019066658 after 0.134252859999\n",
      "before 0.010015511793 after 0.0150232676895\n",
      "before 0.010445931812 after 0.015668897718\n",
      "before 0.0210412824775 after 0.0315619237162\n",
      "before 0.0187758865434 after 0.0281638298152\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0186942192604 after 0.0280413288905\n",
      "before 0.00665202403257 after 0.00997803604885\n",
      "before 0.0283949959174 after 0.0425924938761\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0188138447767 after 0.0282207671651\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0553944718544 after 0.0830917077816\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0186408138096 after 0.0279612207144\n",
      "before 0.0665573930554 after 0.0998360895832\n",
      "before 0.0119808727746 after 0.0179713091619\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.00708689241442 after 0.0106303386216\n",
      "before 0.0154310199726 after 0.0231465299589\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0199937760644 after 0.0299906640966\n",
      "before 0.0755787397734 after 0.11336810966before 0.0507882584589 after 0.0761823876884\n",
      "before 0.0062465767604 after 0.00936986514059\n",
      "\n",
      "before 0.0185290659439 after 0.0277935989159\n",
      "before 0.0188172234647 after 0.028225835197\n",
      "before 0.0 after 0.0\n",
      "before 0.025464704501 after 0.0381970567515\n",
      "before 0.00420356384003 after 0.00630534576004\n",
      "before 0.0268541770527 after 0.0402812655791\n",
      "before 0.00671776000349 after 0.0100766400052\n",
      "before 0.00082637131813 after 0.00123955697719\n",
      "before 0.0 after 0.0\n",
      "before 0.00641233746603 after 0.00961850619904\n",
      "before 0.0113508653524 after 0.0170262980286\n",
      "before 0.0374015720199 after 0.0561023580298\n",
      "before 0.018530484726 after 0.0277957270891\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0393515597966 after 0.0590273396949\n",
      "before 0.033612803889 after 0.0504192058335\n",
      "before 0.0177864607583 after 0.0266796911375\n",
      "before 0.0661984663193 after 0.0992976994789\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0177864607583 after 0.0266796911375\n",
      "before 0.0 after 0.0\n",
      "before 0.00563914511618 after 0.00845871767427\n",
      "before 0.0183466164983 after 0.0275199247474\n",
      "before 0.0225970480831 after 0.0338955721246before 0.00453387901747 after 0.00680081852621\n",
      "before 0.044733824225 after 0.0671007363375\n",
      "\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0176299034053 after 0.0264448551079\n",
      "before 0.0193578610444 after 0.0290367915665\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0134584072694 after 0.0201876109041\n",
      "before 0.0185446585545 after 0.0278169878318\n",
      "before 0.00569413846839 after 0.00854120770259\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0 after 0.0before 0.0584394593484 after 0.0876591890226\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0169349971614 after 0.0254024957421\n",
      "before 0.109615826476 after 0.164423739714\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0417391715931 after 0.0626087573897\n",
      "before 0.0380002332652 after 0.0570003498979\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0175522553762 after 0.0263283830643\n",
      "before 0.0 after 0.0\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0320482938093 after 0.0480724407139\n",
      "before 0.0 after 0.0\n",
      "before 0.0112617405309 after 0.0168926107964\n",
      "before 0.0596368954978 after 0.0894553432467\n",
      "before 0.0360886793772 after 0.0541330190659\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0642782143122 after 0.0964173214683\n",
      "before 0.0107255374609 after 0.0160883061913\n",
      "before 0.00436841998532 after 0.00655262997798\n",
      "before 0.0 after 0.0\n",
      "before 0.0738441169954 after 0.110766175493\n",
      "before 0.0062465767604 after 0.00936986514059\n",
      "before 0.0325395155821 after 0.0488092733731\n",
      "before 0.018968891014 after 0.028453336521\n",
      "before 0.0296947526882 after 0.0445421290323\n",
      "before 0.0796909485543 after 0.119536422831\n",
      "before 0.00420576718955 after 0.00630865078433\n",
      "before 0.128500608841 after 0.192750913261\n",
      "before 0.0642747255818 after 0.0964120883726\n",
      "before 0.0187994456171 after 0.0281991684257\n",
      "before 0.0186877541905 after 0.0280316312857\n",
      "before 0.0366526344197 after 0.0549789516295before 0.00624161226096 after 0.00936241839143\n",
      "before 0.00989917062552 after 0.0148487559383\n",
      "\n",
      "before 0.0172147569204 after 0.0258221353807\n",
      "before 0.00830053629137 after 0.0124508044371\n",
      "before 0.0142148873128 after 0.0213223309692\n",
      "before 0.0269336458632 after 0.0404004687948\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0134584072694 after 0.0201876109041\n",
      "before 0.0287673112696 after 0.0431509669044\n",
      "before 0.0121397831834 after 0.0182096747751\n",
      "before 0.0187176554438 after 0.0280764831657\n",
      "before 0.00142783533939 after 0.00214175300908\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.0137860974647 after 0.020679146197\n",
      "before 0.0299631046033 after 0.044944656905\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.00445770510245 after 0.00668655765367\n",
      "before 0.0 after 0.0\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0249317411888 after 0.0373976117833\n",
      "before 0.0188557898641 after 0.0282836847962\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0656089458574 after 0.0984134187861\n",
      "before 0.0187279077307 after 0.028091861596\n",
      "before 0.0278287190661 after 0.0417430785992\n",
      "before 0.0266996238268 after 0.0400494357402\n",
      "before 0.0188543355317 after 0.0282815032976\n",
      "before 0.0292143599999 after 0.0438215399998\n",
      "before 0.00647609475759 after 0.00971414213638\n",
      "before 0.0330164612158 after 0.0495246918237\n",
      "before 0.0339988708736 after 0.0509983063104\n",
      "before 0.0177864607583 after 0.0266796911375\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0281018322067 after 0.04215274831\n",
      "before 0.0187078719491 after 0.0280618079237\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0309237968125 after 0.0463856952187\n",
      "before 0.0180208609582 after 0.0270312914373\n",
      "before 0.120302053772 after 0.180453080657\n",
      "before 0.0491832555416 after 0.0737748833124\n",
      "before 0.0187817435751 after 0.0281726153626\n",
      "before 0.0177864607583 after 0.0266796911375\n",
      "before 0.0188557898641 after 0.0282836847962\n",
      "before 0.018968891014 after 0.028453336521\n",
      "before 0.0175315790818 after 0.0262973686227\n",
      "before 0.0179404729442 after 0.0269107094164\n",
      "before 0.018968891014 after 0.028453336521\n",
      "before 0.0062465767604 after 0.00936986514059\n",
      "before 0.0297734979566 after 0.0446602469349\n",
      "before 0.018968891014 after 0.028453336521\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0 after 0.0\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.010655362483 after 0.0159830437244\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0188123236977 after 0.0282184855466\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0633175161912 after 0.0949762742869\n",
      "before 0.0132057235567 after 0.019808585335\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0469918101115 after 0.0704877151673\n",
      "before 0.103640987451 after 0.155461481177\n",
      "before 0.011747744167 after 0.0176216162506\n",
      "before 0.0160982337868 after 0.0241473506802\n",
      "before 0.0123658807023 after 0.0185488210534\n",
      "before 0.0131300223102 after 0.0196950334653\n",
      "before 0.0265042330724 after 0.0397563496086\n",
      "before 0.00495441642836 after 0.00743162464253\n",
      "before 0.0540762194873 after 0.081114329231\n",
      "before 0.0228283598827 after 0.034242539824\n",
      "before 0.0367275130082 after 0.0550912695123\n",
      "before 0.0454151132738 after 0.0681226699107\n",
      "before 0.0360246789862 after 0.0540370184793\n",
      "before 0.0275631208911 after 0.0413446813367\n",
      "before 0.0174373778985 after 0.0261560668477\n",
      "before 0.016986775053 after 0.0254801625795\n",
      "before 0.00645773085693 after 0.00968659628539\n",
      "before 0.018960419697 after 0.0284406295455\n",
      "before 0.0224124469439 after 0.0336186704158\n",
      "before 0.0226972817714 after 0.0340459226571\n",
      "before 0.0591502768025 after 0.0887254152037\n",
      "before 0.0187131899385 after 0.0280697849078\n",
      "before 0.0188985122443 after 0.0283477683665\n",
      "before 0.124708299982 after 0.187062449974\n",
      "before 0.0187234939831 after 0.0280852409747\n",
      "before 0.0188985122443 after 0.0283477683665\n",
      "before 0.0175365028979 after 0.0263047543468\n",
      "before 0.0188543355317 after 0.0282815032976\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0160835812379 after 0.0241253718569\n",
      "before 0.0170380000418 after 0.0255570000627\n",
      "before 0.0133994465796 after 0.0200991698694\n",
      "before 0.00269116540345 after 0.00403674810518\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.0304692404213 after 0.045703860632\n",
      "before 0.0291486616409 after 0.0437229924614\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.0317785420361 after 0.0476678130541\n",
      "before 0.01620005269 after 0.024300079035\n",
      "before 0.0107314399209 after 0.0160971598813\n",
      "before 0.0218335845599 after 0.0327503768399\n",
      "before 0.0180802760111 after 0.0271204140166\n",
      "before 0.0166585781399 after 0.0249878672099\n",
      "before 0.0243874516015 after 0.0365811774023\n",
      "before 0.0857961835318 after 0.128694275298\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0179397885523 after 0.0269096828284\n",
      "before 0.0445793662132 after 0.0668690493197\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.00171683627582 after 0.00257525441373\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0205717952238 after 0.0308576928357\n",
      "before 0.0535341449171 after 0.0803012173756\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0117981148589 after 0.0176971722884\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0108328873534 after 0.01624933103\n",
      "before 0.0704169134533 after 0.10562537018\n",
      "before 0.0200887397828 after 0.0301331096742\n",
      "before 0.0167700692412 after 0.0251551038619\n",
      "before 0.0860060376471 after 0.129009056471\n",
      "before 0.0177864607583 after 0.0266796911375\n",
      "before 0.0276058926676 after 0.0414088390014\n",
      "before 0.0177379282361 after 0.0266068923542\n",
      "before 0.0187555371944 after 0.0281333057916\n",
      "before 0.0186941976278 after 0.0280412964417\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.0327336835094 after 0.0491005252641\n",
      "before 0.0082269840831 after 0.0123404761247\n",
      "before 0.00656237563857 after 0.00984356345786\n",
      "before 0.0152875639159 after 0.0229313458738\n",
      "before 0.00948735211137 after 0.0142310281671\n",
      "before 0.050306733153 after 0.0754600997295\n",
      "before 0.00505298063418 after 0.00757947095127\n",
      "before 0.00621714289916 after 0.00932571434874\n",
      "before 0.008202987599 after 0.0123044813985\n",
      "before 0.027000420072 after 0.0405006301079\n",
      "before 0.0139246481914 after 0.0208869722871\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0211945000414 after 0.0317917500621\n",
      "before 0.0433553616491 after 0.0650330424736\n",
      "before 0.0206547883179 after 0.0309821824768\n",
      "before 0.0521519168151 after 0.0782278752227\n",
      "before 0.00621714289916 after 0.00932571434874\n",
      "before 0.00674018554532 after 0.010110278318\n",
      "before 0.01858244777 after 0.0278736716551\n",
      "before 0.0246063068873 after 0.0369094603309\n",
      "before 0.0222069651378 after 0.0333104477067\n",
      "before 0.0 after 0.0\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.030620368491 after 0.0459305527365\n",
      "before 0.0177621822386 after 0.0266432733579\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0322725636127 after 0.0484088454191\n",
      "before 0.00993769244835 after 0.0149065386725\n",
      "before 0.00649440971485 after 0.00974161457227\n",
      "before 0.110486692629 after 0.165730038943\n",
      "before 0.00656981096006 after 0.00985471644009\n",
      "before 0.018968891014 after 0.028453336521\n",
      "before 0.00649440971485 after 0.00974161457227\n",
      "before 0.0188123236977 after 0.0282184855466\n",
      "before 0.00361066170472 after 0.00541599255708\n",
      "before 0.018882457109 after 0.0283236856635\n",
      "before 0.00787711954793 after 0.0118156793219\n",
      "before 0.0143624531549 after 0.0215436797324\n",
      "before 0.0187180880221 after 0.0280771320331\n",
      "before 0.0 after 0.0\n",
      "before 0.0710907804672 after 0.106636170701\n",
      "before 0.0188543355317 after 0.0282815032976\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0459675435102 after 0.0689513152653\n",
      "before 0.0165798100256 after 0.0248697150383\n",
      "before 0.018882457109 after 0.0283236856635\n",
      "before 0.0113413686738 after 0.0170120530107\n",
      "before 0.0188123236977 after 0.0282184855466\n",
      "before 0.0263210656422 after 0.0394815984632\n",
      "before 0.00519728197889 after 0.00779592296833\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.0126751652056 after 0.0190127478083\n",
      "before 0.0312128379344 after 0.0468192569015\n",
      "before 0.0188123236977 after 0.0282184855466\n",
      "before 0.0180839492416 after 0.0271259238623\n",
      "before 0.0171811346395 after 0.0257717019592\n",
      "before 0.145468517796 after 0.218202776694\n",
      "before 0.0321265182437 after 0.0481897773656\n",
      "before 0.0472621665788 after 0.0708932498682\n",
      "before 0.0188543355317 after 0.0282815032976\n",
      "before 0.0188123236977 after 0.0282184855466\n",
      "before 0.0183246969561 after 0.0274870454342\n",
      "before 0.0184613257562 after 0.0276919886342\n",
      "before 0.00324918681215 after 0.00487378021823\n",
      "before 0.00624172636665 after 0.00936258954998\n",
      "before 0.00725341243626 after 0.0108801186544\n",
      "before 0.0 after 0.0\n",
      "before 0.031795946808 after 0.047693920212\n",
      "before 0.0675427124158 after 0.101314068624\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0169083687305 after 0.0253625530958\n",
      "before 0.0339315251792 after 0.0508972877688\n",
      "before 0.0289164793166 after 0.0433747189748\n",
      "before 0.0189526559945 after 0.0284289839917\n",
      "before 0.0272481090374 after 0.040872163556\n",
      "before 0.0350599297617 after 0.0525898946426\n",
      "before 0.00680874813959 after 0.0102131222094\n",
      "before 0.0 after 0.0\n",
      "before 0.0238315796373 after 0.0357473694559\n",
      "before 0.00395571426655 after 0.00593357139983\n",
      "before 0.0188985122443 after 0.0283477683665\n",
      "before 0.00802919741127 after 0.0120437961169\n",
      "before 0.026168782718 after 0.0392531740769\n",
      "before 0.0188985122443 after 0.0283477683665\n",
      "before 0.0120134648021 after 0.0180201972032\n",
      "before 0.00368573486309 after 0.00552860229463\n",
      "before 0.0188985122443 after 0.0283477683665\n",
      "before 0.0176578361036 after 0.0264867541553\n",
      "before 0.0446960407533 after 0.0670440611299\n",
      "before 0.0187239608282 after 0.0280859412423\n",
      "before 0.120302053772 after 0.180453080657\n",
      "before 0.0085691214476 after 0.0128536821714\n",
      "before 0.0272830937922 after 0.0409246406884\n",
      "before 0.0289059884681 after 0.0433589827022\n",
      "before 0.0157811487368 after 0.0236717231053\n",
      "before 0.0197412986429 after 0.0296119479644\n",
      "before 0.0181267092514 after 0.0271900638771\n",
      "before 0.0253170429992 after 0.0379755644989\n",
      "before 0.0230698761825 after 0.0346048142737\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0135699820948 after 0.0203549731422\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.018882457109 after 0.0283236856635\n",
      "before 0.010373206918 after 0.015559810377\n",
      "before 0.0355530625771 after 0.0533295938656\n",
      "before 0.00818938802086 after 0.0122840820313\n",
      "before 0.0175015882036 after 0.0262523823054\n",
      "before 0.0377850388315 after 0.0566775582473\n",
      "before 0.0446597482589 after 0.0669896223884\n",
      "before 0.0137857170482 after 0.0206785755723\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0136124223568 after 0.0204186335352\n",
      "before 0.0187612232148 after 0.0281418348222\n",
      "before 0.0305884555573 after 0.0458826833359\n",
      "before 0.00287135991847 after 0.0043070398777\n",
      "before 0.0164085918137 after 0.0246128877205\n",
      "before 0.0578260233913 after 0.0867390350869\n",
      "before 0.0125809358407 after 0.018871403761\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0456823199321 after 0.0685234798982\n",
      "before 0.0379983431203 after 0.0569975146805\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0306845085948 after 0.0460267628921\n",
      "before 0.0220919434318 after 0.0331379151477\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0190837258291 after 0.0286255887437\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0115753375879 after 0.0173630063819\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.000814234575361 after 0.00122135186304\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0 after 0.0before 0.0299325679849 after 0.0448988519773\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "\n",
      "before 0.018968891014 after 0.028453336521\n",
      "before 0.0260509602701 after 0.0390764404052\n",
      "before 0.0 after 0.0\n",
      "before 0.034649021247 after 0.0519735318705\n",
      "before 0.0213953921447 after 0.0320930882171\n",
      "before 0.0 after 0.0\n",
      "before 0.0188123236977 after 0.0282184855466\n",
      "before 0.017628000136 after 0.0264420002039\n",
      "before 0.0 after 0.0\n",
      "before 0.0417701635782 after 0.0626552453673\n",
      "before 0.0394193185372 after 0.0591289778058\n",
      "before 0.0 after 0.0\n",
      "before 0.0131483081084 after 0.0197224621626\n",
      "before 0.0791738695553 after 0.118760804333\n",
      "before 0.0 after 0.0\n",
      "before 0.0185784542141 after 0.0278676813212\n",
      "before 0.0356253443915 after 0.0534380165872\n",
      "before 0.0 after 0.0\n",
      "before 0.0117332999442 after 0.0175999499163\n",
      "before 0.0188985122443 after 0.0283477683665\n",
      "before 0.0 after 0.0\n",
      "before 0.0146730458342 after 0.0220095687513\n",
      "before 0.00829015124205 after 0.0124352268631\n",
      "before 0.0 after 0.0\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.00334638742381 after 0.00501958113572\n",
      "before 0.0242833621589 after 0.0364250432383\n",
      "before 0.0106591574666 after 0.0159887361999\n",
      "before 0.0591068888754 after 0.0886603333131\n",
      "before 0.0 after 0.0\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0188557898641 after 0.0282836847962\n",
      "before 0.0 after 0.0\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.00906109389996 after 0.0135916408499\n",
      "before 0.0 after 0.0\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0134756072686 after 0.0202134109028\n",
      "before 0.0 after 0.0\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0064487910945 after 0.00967318664175\n",
      "before 0.0 after 0.0\n",
      "before 0.00912698305686 after 0.0136904745853\n",
      "before 0.0187567783397 after 0.0281351675096\n",
      "before 0.0 after 0.0\n",
      "before 0.0264901727342 after 0.0397352591013\n",
      "before 0.0188985122443 after 0.0283477683665\n",
      "before 0.0 after 0.0\n",
      "before 0.0187852195293 after 0.0281778292939\n",
      "before 0.0701634766356 after 0.105245214953\n",
      "before 0.0 after 0.0\n",
      "before 0.0188557898641 after 0.0282836847962\n",
      "before 0.018882457109 after 0.0283236856635\n",
      "before 0.0 after 0.0\n",
      "before 0.041613114061 after 0.0624196710915\n",
      "before 0.018882457109 after 0.0283236856635\n",
      "before 0.0479538774734 after 0.0719308162101\n",
      "before 0.0155288507552 after 0.0232932761328\n",
      "before 0.0180855634876 after 0.0271283452314\n",
      "before 0.0 after 0.0\n",
      "before 0.00846161558019 after 0.0126924233703\n",
      "before 0.0188123236977 after 0.0282184855466\n",
      "before 0.0 after 0.0\n",
      "before 0.00832046682785 after 0.0124807002418\n",
      "before 0.015977759828 after 0.0239666397419\n",
      "before 0.0 after 0.0\n",
      "before 0.0102145024075 after 0.0153217536113\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0 after 0.0\n",
      "before 0.0273303255918 after 0.0409954883876\n",
      "before 0.0171459116094 after 0.0257188674141\n",
      "before 0.0 after 0.0\n",
      "before 0.0105903407581 after 0.0158855111372\n",
      "before 0.0145733681363 after 0.0218600522044\n",
      "before 0.0 after 0.0\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0176335279138 after 0.0264502918706\n",
      "before 0.0 after 0.0\n",
      "before 0.02878754026 after 0.04318131039\n",
      "before 0.00962513632114 after 0.0144377044817\n",
      "before 0.0 after 0.0\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.00608235818812 after 0.00912353728218\n",
      "before 0.0 after 0.0\n",
      "before 0.0121465013889 after 0.0182197520834\n",
      "before 0.0188543355317 after 0.0282815032976\n",
      "before 0.0 after 0.0\n",
      "before 0.01564434487 after 0.023466517305\n",
      "before 0.0188543355317 after 0.0282815032976\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0before 0.0777497389293 after 0.116624608394\n",
      "before 0.01905460387 after 0.028581905805\n",
      "\n",
      "before 0.0291134428356 after 0.0436701642534\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0177779671133 after 0.02666695067\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0103722434767 after 0.015558365215\n",
      "before 0.0488705685354 after 0.0733058528031\n",
      "before 0.0 after 0.0\n",
      "before 0.0163467109773 after 0.0245200664659\n",
      "before 0.0625573602179 after 0.0938360403268\n",
      "before 0.0 after 0.0\n",
      "before 0.0522478397382 after 0.0783717596073\n",
      "before 0.0 after 0.0\n",
      "before 0.0169398869451 after 0.0254098304176\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0 after 0.0\n",
      "before 0.0192243664848 after 0.0288365497272\n",
      "before 0.018968891014 after 0.028453336521\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0430914075099 after 0.0646371112648\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0761902941699 after 0.114285441255\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0213321659987 after 0.031998248998\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0189526559945 after 0.0284289839917\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0 after 0.0\n",
      "before 0.0149521348034 after 0.0224282022051\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0188985122443 after 0.0283477683665\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0344443129155 after 0.0516664693733\n",
      "before 0.0219780859236 after 0.0329671288854\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0514903766626 after 0.077235564994\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0187852195293 after 0.0281778292939\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0258167392651 after 0.0387251088976\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00681062913761 after 0.0102159437064\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00650098431079 after 0.00975147646618\n",
      "before 0.045897636173 after 0.0688464542595\n",
      "before 0.0 after 0.0\n",
      "before 0.0187866579252 after 0.0281799868878\n",
      "before 0.0177717599665 after 0.0266576399497\n",
      "before 0.0 after 0.0\n",
      "before 0.0285611437823 after 0.0428417156734\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00571710302534 after 0.00857565453802\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00459655726414 after 0.00689483589621\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0 after 0.0\n",
      "before 0.0116694191831 after 0.0175041287747\n",
      "before 0.0188010276987 after 0.028201541548\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.279279508943 after 0.418919263415\n",
      "before 0.0459548027463 after 0.0689322041195\n",
      "before 0.0 after 0.0\n",
      "before 0.050192064732 after 0.0752880970981\n",
      "before 0.0 after 0.0\n",
      "before 0.0153593635399 after 0.0230390453098\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0249391364532 after 0.0374087046799\n",
      "before 0.0233717398925 after 0.0350576098387\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.018648536983 after 0.0279728054745\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0135719809198 after 0.0203579713797\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0444935139434 after 0.0667402709151\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0523859690703 after 0.0785789536054\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.018079362074 after 0.027119043111\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0251396832913 after 0.0377095249369\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0157843156321 after 0.0236764734481\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0105365319462 after 0.0158047979193\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0415086746775 after 0.0622630120163\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0172061102582 after 0.0258091653873\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0223017086301 after 0.0334525629451\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0182374692273 after 0.0273562038409\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.088069105397 after 0.132103658096\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.011546245555 after 0.0173193683325\n",
      "before 0.0158635810484 after 0.0237953715726\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0920418082599 after 0.13806271239\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0148934188882 after 0.0223401283323\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0645114529465 after 0.0967671794198\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0116694191831 after 0.0175041287747\n",
      "before 0.0 after 0.0\n",
      "before 0.025975505327 after 0.0389632579905\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0282891626494 after 0.0424337439741\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.019023050829 after 0.0285345762435\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.033794069484 after 0.050691104226\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0415956223804 after 0.0623934335706\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0277848312314 after 0.041677246847\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0218952067966 after 0.0328428101949\n",
      "before 0.111239994487 after 0.16685999173\n",
      "before 0.0 after 0.0\n",
      "before 0.0194682368042 after 0.0292023552063\n",
      "before 0.0225860087985 after 0.0338790131978\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0473429047571 after 0.0710143571357\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0161141523739 after 0.0241712285609\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0102076219522 after 0.0153114329283\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00740290643571 after 0.0111043596536\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.465393526743 after 0.698090290115\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0633360720843 after 0.0950041081265\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0105313274988 after 0.0157969912482\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.017843170203 after 0.0267647553045\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00335305623217 after 0.00502958434825\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.024321987757 after 0.0364829816355\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0282922756272 after 0.0424384134408\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0101096567026 after 0.0151644850539\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0443709829165 after 0.0665564743747\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0103588428535 after 0.0155382642802\n",
      "before 0.0726698827901 after 0.109004824185\n",
      "before 0.0 after 0.0\n",
      "before 0.13646059347 after 0.204690890206\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0274052385936 after 0.0411078578905\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0135410354725 after 0.0203115532087\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0591159140221 after 0.0886738710331\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0185369458961 after 0.0278054188442\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0101068547982 after 0.0151602821974\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00526141326481 after 0.00789211989721\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.446326638447 after 0.66948995767\n",
      "before 0.0 after 0.0\n",
      "before 0.0656246029721 after 0.0984369044581\n",
      "before 0.0389837530927 after 0.0584756296391\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00955699773481 after 0.0143354966022\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.019848214539 after 0.0297723218084\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.00418687837066 after 0.006280317556\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.028173164046 after 0.042259746069\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0341843440091 after 0.0512765160137\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0100098566377 after 0.0150147849566\n",
      "before 0.0170250071133 after 0.02553751067\n",
      "before 0.0525074160449 after 0.0787611240673\n",
      "before 0.0 after 0.0\n",
      "before 0.0363961423641 after 0.0545942135461\n",
      "before 0.0 after 0.0\n",
      "before 0.0119293770151 after 0.0178940655226\n",
      "before 0.0 after 0.0\n",
      "before 0.113770356344 after 0.170655534516\n",
      "before 0.0 after 0.0\n",
      "before 0.0228483819482 after 0.0342725729224\n",
      "before 0.0 after 0.0\n",
      "before 0.0492863062724 after 0.0739294594087\n",
      "before 0.0 after 0.0\n",
      "before 0.0154546809925 after 0.0231820214888\n",
      "before 0.0 after 0.0\n",
      "before 0.0084263188564 after 0.0126394782846\n",
      "before 0.0 after 0.0\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0 after 0.0\n",
      "before 0.00438656206006 after 0.0065798430901\n",
      "before 0.0 after 0.0\n",
      "before 0.455141705831 after 0.682712558746\n",
      "before 0.0 after 0.0\n",
      "before 0.0638978013685 after 0.0958467020527\n",
      "before 0.0 after 0.0\n",
      "before 0.00796789029139 after 0.0119518354371\n",
      "before 0.0 after 0.0\n",
      "before 0.0165479160208 after 0.0248218740312\n",
      "before 0.0 after 0.0\n",
      "before 0.00349069743936 after 0.00523604615903\n",
      "before 0.0 after 0.0\n",
      "before 0.0234886191782 after 0.0352329287673\n",
      "before 0.0 after 0.0\n",
      "before 0.0285002790945 after 0.0427504186417\n",
      "before 0.0 after 0.0\n",
      "before 0.00834544924412 after 0.0125181738662\n",
      "before 0.0 after 0.0\n",
      "before 0.0437766484977 after 0.0656649727465\n",
      "before 0.0 after 0.0\n",
      "before 0.00994579782668 after 0.01491869674\n",
      "before 0.0 after 0.0\n",
      "before 0.135797002849 after 0.203695504274\n",
      "before 0.0 after 0.0\n",
      "before 0.0272719703816 after 0.0409079555724\n",
      "before 0.0 after 0.0\n",
      "before 0.0134751871282 after 0.0202127806923\n",
      "before 0.0 after 0.0\n",
      "before 0.0588284408028 after 0.0882426612041\n",
      "before 0.0631570507972 after 0.0947355761958\n",
      "before 0.0184468030708 after 0.0276702046062\n",
      "before 0.0 after 0.0\n",
      "before 0.0100577064406 after 0.0150865596609\n",
      "before 0.0 after 0.0\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0 after 0.0\n",
      "before 0.00523582767699 after 0.00785374151549\n",
      "before 0.0 after 0.0\n",
      "before 0.486449816824 after 0.729674725236\n",
      "before 0.0 after 0.0\n",
      "before 0.0387941800279 after 0.0581912700418\n",
      "before 0.0 after 0.0\n",
      "before 0.00951052326256 after 0.0142657848938\n",
      "before 0.0 after 0.0\n",
      "before 0.0197516951799 after 0.0296275427699\n",
      "before 0.0 after 0.0\n",
      "before 0.00416651811025 after 0.00624977716538\n",
      "before 0.0 after 0.0\n",
      "before 0.0280361615096 after 0.0420542422643\n",
      "before 0.0 after 0.0\n",
      "before 0.0340181098641 after 0.0510271647962\n",
      "before 0.0 after 0.0\n",
      "before 0.00996117996987 after 0.0149417699548\n",
      "before 0.0 after 0.0\n",
      "before 0.0522520791161 after 0.0783781186742\n",
      "before 0.0 after 0.0\n",
      "before 0.0284981552974 after 0.0427472329461\n",
      "before 0.0 after 0.0\n",
      "before 0.0118713659622 after 0.0178070489432\n",
      "before 0.0328614923188 after 0.0492922384781\n",
      "before 0.119419386568 after 0.179129079852\n",
      "before 0.0 after 0.0\n",
      "before 0.0269013454667 after 0.0403520182001\n",
      "before 0.0 after 0.0\n",
      "before 0.00759963656184 after 0.0113994548428\n",
      "before 0.0 after 0.0\n",
      "before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0 after 0.0\n",
      "55577138ff006709640039ed1f736563726574. for 17368092 similarity is 0.698090290115\n",
      "55577138ff006709640039ed1f736563726574. for 16710515 similarity is 0.565351214989\n",
      "55577138ff006709640039ed1f736563726574. for 17493514 similarity is 0.289806375476\n",
      "55577138ff006709640039ed1f736563726574. for 17493541 similarity is 0.271265765139\n",
      "55577138ff006709640039ed1f736563726574. for 17493498 similarity is 0.271265765139\n",
      "55577138ff006709640039ed1f736563726574. for 17192222 similarity is 0.211290724198\n",
      "55577138ff006709640039ed1f736563726574. for 15623566 similarity is 0.209434365519\n",
      "55577138ff006709640039ed1f736563726574. for 15655585 similarity is 0.204181171088\n",
      "55577138ff006709640039ed1f736563726574. for 17493099 similarity is 0.18369293341\n",
      "55577138ff006709640039ed1f736563726574. for 17493311 similarity is 0.16685999173\n",
      "55577138ff006709640039ed1f736563726574. for 17493236 similarity is 0.164108594048\n",
      "55577138ff006709640039ed1f736563726574. for 17493378 similarity is 0.161450713398\n",
      "55577138ff006709640039ed1f736563726574. for 12827505 similarity is 0.134863318802\n",
      "55577138ff006709640039ed1f736563726574. for 17203318 similarity is 0.127263480893\n",
      "55577138ff006709640039ed1f736563726574. for 17493496 similarity is 0.12494932842\n",
      "55577138ff006709640039ed1f736563726574. for 17493508 similarity is 0.120960938749\n",
      "55577138ff006709640039ed1f736563726574. for 17493529 similarity is 0.118962662421\n",
      "55577138ff006709640039ed1f736563726574. for 17493743 similarity is 0.102550586939\n",
      "55577138ff006709640039ed1f736563726574. for 13612196 similarity is 0.0950041081265\n",
      "55577138ff006709640039ed1f736563726574. for 16930097 similarity is 0.0914615623877\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493514 similarity is 0.218202776694before 0.00424485080862 after 0.00636727621294\n",
      "before 0.0 after 0.0\n",
      "before 0.0345351631262 after 0.0518027446893\n",
      "before 0.0 after 0.0\n",
      "before 0.0144844600944 after 0.0217266901416\n",
      "before 0.0076600003749 after 0.0114900005623\n",
      "before 0.0442146953623 after 0.0663220430435\n",
      "before 0.00468377524264 after 0.00702566286397\n",
      "before 0.00914158361917 after 0.0137123754288\n",
      "before 0.00463271679426 after 0.00694907519139\n",
      "before 0.02250596111 after 0.033758941665\n",
      "before 0.0780492887659 after 0.117073933149\n",
      "before 0.0185972394867 after 0.02789585923\n",
      "before 0.00414624441679 after 0.00621936662518\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0183290591624 after 0.0274935887436\n",
      "before 0.0173245931792 after 0.0259868897689\n",
      "before 0.0303003617829 after 0.0454505426743\n",
      "before 0.0155890832975 after 0.0233836249463\n",
      "before 0.0157925959782 after 0.0236888939674\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0277669040402 after 0.0416503560604\n",
      "before 0.0014083655035 after 0.00211254825525\n",
      "before 0.0118496502629 after 0.0177744753944\n",
      "before 0.0144241631369 after 0.0216362447053\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0185461912184 after 0.0278192868276\n",
      "before 0.0186249776007 after 0.027937466401\n",
      "before 0.0185768375118 after 0.0278652562676\n",
      "before 0.0 after 0.0\n",
      "before 0.0302028318654 after 0.0453042477981\n",
      "before 0.0359522325561 after 0.0539283488341\n",
      "before 0.0185587261384 after 0.0278380892077\n",
      "before 0.045950827775 after 0.0689262416625\n",
      "before 0.056789937756 after 0.085184906634\n",
      "before 0.00428944743793 after 0.0064341711569\n",
      "before 0.0284050845322 after 0.0426076267982\n",
      "before 0.00904058227125 after 0.0135608734069\n",
      "before 0.0714862509604 after 0.107229376441\n",
      "before 0.0145892822644 after 0.0218839233965\n",
      "before 0.0034854590443 after 0.00522818856645\n",
      "before 0.0762667881846 after 0.114400182277\n",
      "before 0.0 after 0.0\n",
      "before 0.0156410302548 after 0.0234615453822\n",
      "before 0.00794690003397 after 0.011920350051\n",
      "before 0.019635511024 after 0.0294532665361\n",
      "before 0.0412781941067 after 0.06191729116\n",
      "before 0.0217342760445 after 0.0326014140667\n",
      "before 0.0154774118914 after 0.0232161178371\n",
      "before 0.01855730086 after 0.0278359512901\n",
      "before 0.018598673988 after 0.027898010982\n",
      "before 0.0661675108103 after 0.0992512662154\n",
      "before 0.0105159391156 after 0.0157739086734\n",
      "before 0.0146187863894 after 0.0219281795842\n",
      "before 0.0326511677958 after 0.0489767516937\n",
      "before 0.0190902137982 after 0.0286353206972\n",
      "before 0.0643413499762 after 0.0965120249643\n",
      "before 0.0188543355317 after 0.0282815032976\n",
      "before 0.00420356384003 after 0.00630534576004\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.01858244777 after 0.0278736716551\n",
      "before 0.0175640956275 after 0.0263461434412\n",
      "before 0.0257101065384 after 0.0385651598075\n",
      "before 0.0158045933286 after 0.023706889993\n",
      "before 0.0160109194541 after 0.0240163791812\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.0281507653771 after 0.0422261480656\n",
      "before 0.00142783533939 after 0.00214175300908\n",
      "before 0.0120134648021 after 0.0180201972032\n",
      "before 0.0146235688227 after 0.0219353532341\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.0188025815507 after 0.028203872326\n",
      "before 0.018882457109 after 0.0283236856635\n",
      "before 0.0188336515113 after 0.028250477267\n",
      "before 0.0 after 0.0\n",
      "before 0.030620368491 after 0.0459305527365\n",
      "before 0.0364492513102 after 0.0546738769652\n",
      "before 0.0188152897587 after 0.0282229346381\n",
      "before 0.0400004891317 after 0.0600007336976\n",
      "before 0.0520239327467 after 0.0780358991201\n",
      "before 0.00434874655984 after 0.00652311983976\n",
      "before 0.028797768344 after 0.043196652516\n",
      "before 0.00916556307542 after 0.0137483446131\n",
      "before 0.0724745069005 after 0.108711760351\n",
      "before 0.0147909706263 after 0.0221864559395\n",
      "before 0.00353364349318 after 0.00530046523976\n",
      "before 0.0747245557498 after 0.112086833625\n",
      "before 0.0 after 0.0\n",
      "before 0.0158572584225 after 0.0237858876337\n",
      "before 0.00805676131581 after 0.0120851419737\n",
      "before 0.0199069605706 after 0.0298604408559\n",
      "before 0.0418488411889 after 0.0627732617833\n",
      "before 0.0220347397996 after 0.0330521096994\n",
      "before 0.0156913781302 after 0.0235370671953\n",
      "before 0.0188138447767 after 0.0282207671651\n",
      "before 0.0188557898641 after 0.0282836847962\n",
      "before 0.0636739960232 after 0.0955109940348\n",
      "before 0.0106613158721 after 0.0159919738081\n",
      "before 0.0148208826288 after 0.0222313239433\n",
      "before 0.0221851519376 after 0.0332777279065\n",
      "before 0.019354124927 after 0.0290311873904\n",
      "before 0.110109381758 after 0.165164072637\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0415086746775 after 0.0622630120163\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0438647461185 after 0.0657971191777\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0518610288777 after 0.0777915433166\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "before 0.0 after 0.0\n",
      "\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493828 similarity is 0.192750913261\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493378 similarity is 0.187062449974\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493541 similarity is 0.180453080657\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493498 similarity is 0.180453080657\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493496 similarity is 0.165730038943\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493435 similarity is 0.155461481177\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17143301 similarity is 0.147219843585\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493259 similarity is 0.129009056471\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 16711880 similarity is 0.128694275298\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17192222 similarity is 0.118760804333\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493224 similarity is 0.116624608394\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17427521 similarity is 0.114285441255\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17258909 similarity is 0.112086833625\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493367 similarity is 0.110766175493\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493513 similarity is 0.108711760351\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 14457387 similarity is 0.106636170701\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493099 similarity is 0.10562537018\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 12827505 similarity is 0.105245214953\n",
      "c13f3c94ff023cf7d20039ed1f30766857326a. for 17493529 similarity is 0.101314068624\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 10520241 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17493552 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17493553 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17493551 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 15515841 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 16632858 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 15516607 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17493300 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17427661 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 16992235 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 16992230 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 16898230 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17258726 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17493100 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17140490 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17211195 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 16616468 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 16089628 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17400020 similarity is 0.0\n",
      "a588f755ff02418ca00039ed1f5a75387a3955. for 17403291 similarity is 0.0\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17368092 similarity is 0.682712558746\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 16710515 similarity is 0.539263453111\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493514 similarity is 0.296642700882\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493541 similarity is 0.277287028047\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493498 similarity is 0.277287028047\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 15623566 similarity is 0.216820730839\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17192222 similarity is 0.210409691764\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 15655585 similarity is 0.210055472902\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493099 similarity is 0.190308861832\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493311 similarity is 0.170655534516\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493378 similarity is 0.168078180865\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493236 similarity is 0.150360688369\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 12827505 similarity is 0.138825379814\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493508 similarity is 0.124920130193\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493529 similarity is 0.123157549227\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493496 similarity is 0.122738022515\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17203318 similarity is 0.122366286351\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493743 similarity is 0.106760234975\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 13612196 similarity is 0.0958467020527\n",
      "3fae6c58ff0213a0350039ed1f657568484248. for 17493663 similarity is 0.0930308147767\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493514 similarity is 0.241036631958\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493378 similarity is 0.210414872488\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493496 similarity is 0.208694616677\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493541 similarity is 0.204077016649\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493498 similarity is 0.204077016649\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493828 similarity is 0.202220584574\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493435 similarity is 0.153341622225\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17143301 similarity is 0.145212366871\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17427521 similarity is 0.134252859999\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17192222 similarity is 0.132523683609\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493099 similarity is 0.12902283924\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493259 similarity is 0.127249900433\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 16711880 similarity is 0.126939411588\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 12827505 similarity is 0.124952261478\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493311 similarity is 0.117073933149\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493224 similarity is 0.11503432559\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493529 similarity is 0.114728631926\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17258909 similarity is 0.114400182277\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 17493367 similarity is 0.10925577776\n",
      "848f3de0ff000c40da0039ed1f736563726574. for 16930097 similarity is 0.108203959366\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17492991 similarity is 0.418919263415\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493311 similarity is 0.165164072637\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 16786723 similarity is 0.13806271239\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493192 similarity is 0.132103658096\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493596 similarity is 0.109004824185\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493187 similarity is 0.0984369044581\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17367799 similarity is 0.0967671794198\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17492978 similarity is 0.0966671644098\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493663 similarity is 0.0947355761958\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17203318 similarity is 0.0938360403268\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17367895 similarity is 0.0909615451155\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 15623566 similarity is 0.0785789536054\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493109 similarity is 0.0777915433166\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17427496 similarity is 0.0752880970981\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17140936 similarity is 0.0733058528031\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493348 similarity is 0.0719308162101\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493366 similarity is 0.0689322041195\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493504 similarity is 0.0688464542595\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17493484 similarity is 0.0667402709151\n",
      "04589be6ff01f2fef60039ed1f56534e424e42. for 17143392 similarity is 0.0657971191777\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17368092 similarity is 0.729674725236\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 16710515 similarity is 0.529709483238\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493514 similarity is 0.31977369964\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493541 similarity is 0.309744864908\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493498 similarity is 0.309744864908\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493702 similarity is 0.306639540752\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 15623566 similarity is 0.258798568886\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 15655585 similarity is 0.24031215585\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17192222 similarity is 0.234298091443\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493099 similarity is 0.227153837632\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493311 similarity is 0.203695504274\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493378 similarity is 0.192039155522\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 12827505 similarity is 0.165702834235\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493187 similarity is 0.165638558707\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493508 similarity is 0.149105442057\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493496 similarity is 0.146500864801\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493529 similarity is 0.13972753258\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17203318 similarity is 0.139151073858\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17493236 similarity is 0.136013188874\n",
      "f11c5aeaff018d389a0039ed1f3245516a4a6d. for 17427521 similarity is 0.13523928004\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17368092 similarity is 0.66948995767\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 16710515 similarity is 0.526690969095\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493514 similarity is 0.321336317545\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493541 similarity is 0.31125847554\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493498 similarity is 0.31125847554\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493702 similarity is 0.308137976793\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 15623566 similarity is 0.260063223477\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 15655585 similarity is 0.241486474056\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17192222 similarity is 0.235443021102\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493099 similarity is 0.228263855917\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493311 similarity is 0.204690890206\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493378 similarity is 0.192977581112\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 12827505 similarity is 0.176864273607\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493187 similarity is 0.166447974171\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493508 similarity is 0.149834066186\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493496 similarity is 0.147216761307\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493529 similarity is 0.140410330272\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17203318 similarity is 0.139831054606\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17493236 similarity is 0.136677835918\n",
      "95dd84faff01e0b6a90039ed1f6d3657666945. for 17427521 similarity is 0.135900145273\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493348 similarity is 0.232518075215\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493366 similarity is 0.222824990271\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493311 similarity is 0.179129079852\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493663 similarity is 0.133709758537\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493743 similarity is 0.126392178813\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493378 similarity is 0.117476368954\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17445920 similarity is 0.117390855611\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17445924 similarity is 0.117196464875\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17265409 similarity is 0.116645324356\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 16930097 similarity is 0.116135394703\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17445934 similarity is 0.113035769851\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17427499 similarity is 0.109490192888\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17203318 similarity is 0.106339288072\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493750 similarity is 0.103967670267\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 15623566 similarity is 0.100659204087\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493236 similarity is 0.097789038254\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 16943974 similarity is 0.0948406702591\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493514 similarity is 0.0942811957657\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17192222 similarity is 0.0889083909421\n",
      "4527975dff02f59d5f0039ed1f534356744c6e. for 17493508 similarity is 0.083830316759\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493828 similarity is 0.189192574961\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17156248 similarity is 0.173509337087\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17140490 similarity is 0.164423739714\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493615 similarity is 0.148990294192\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493204 similarity is 0.123214116988\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 16999871 similarity is 0.119536422831\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17258682 similarity is 0.114378477394\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17492935 similarity is 0.11336810966\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17265409 similarity is 0.0992976994789\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493725 similarity is 0.0964173214683\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493568 similarity is 0.0964120883726\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493248 similarity is 0.0928106053348\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493579 similarity is 0.0926009746813\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17309377 similarity is 0.0914763763591\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17186378 similarity is 0.0821428622034\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493464 similarity is 0.0792177579485\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 16954698 similarity is 0.0662556422837\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493575 similarity is 0.0660512835539\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17493702 similarity is 0.0619437421927\n",
      "d7c90e2eff02418ca50039ed1f356b4e504642. for 17492969 similarity is 0.0590273396949\n",
      "total time 24.7118980885 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "#recommender2\n",
    "import pickle\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import heapq\n",
    "import numpy\n",
    "import ConfigParser\n",
    "import MySQLdb\n",
    "import json\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import httplib\n",
    "import re \n",
    "import Stemmer\n",
    "import time\n",
    "import datetime\n",
    "import redis\n",
    "import threading\n",
    "from multiprocessing import Pool\n",
    "\n",
    "print 'Start at {}'.format(datetime.datetime.now())\n",
    "start_time = time.time()\n",
    "r = redis.StrictRedis(host='localhost', port=6379, db=0)\n",
    "config = ConfigParser.ConfigParser()\n",
    "config.readfp(open('my.cfg'))\n",
    "\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "if r1.status != 200:\n",
    "    conn.close()\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "conn.close()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "#areas\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/areas\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "if r1.status != 200:\n",
    "    conn.close()\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    conn.request(\"GET\", \"https://api.hh.ru/areas\", headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "areas = r1.read()\n",
    "conn.close()\n",
    "areas_json = json.loads(areas)\n",
    "areas_map = {}\n",
    "def build_areas_map(areas, areas_map):\n",
    "    for area in areas:\n",
    "        if area['id'] == '1':#msk\n",
    "            parent_id = '2019'\n",
    "        elif area['id'] == '2':#spb\n",
    "            parent_id = '145'\n",
    "        elif area['id'] == '115':#kiev\n",
    "            parent_id = '2164'\n",
    "        elif area['id'] == '1002':#minsk\n",
    "            parent_id = '2237'\n",
    "        else:\n",
    "            parent_id = area['parent_id']\n",
    "        areas_map[area['id']] = parent_id\n",
    "        build_areas_map(area['areas'], areas_map)\n",
    "        \n",
    "build_areas_map(areas_json, areas_map)\n",
    "  \n",
    "with open( \"count_vectorizer.p\", \"rb\" ) as f:\n",
    "    count_vectorizer = pickle.load(f)\n",
    "    \n",
    "with open( \"tfidf_transformer.p\", \"rb\" ) as f:\n",
    "    tfidf_transformer = pickle.load(f)\n",
    "\n",
    "def get_resumes():\n",
    "    db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                     port=config.getint('mysqld', 'port'), \n",
    "                     user=config.get('mysqld', 'user'), \n",
    "                     passwd=config.get('mysqld', 'password'), \n",
    "                     db=config.get('mysqld', 'database') )\n",
    "    db.autocommit(True)\n",
    "    db.set_character_set('utf8')\n",
    "    cursor = db.cursor()\n",
    "    cursor.execute('SET NAMES utf8;')\n",
    "    cursor.execute('SET CHARACTER SET utf8;')\n",
    "    cursor.execute('SET character_set_connection=utf8;')\n",
    "    cursor.close()\n",
    "\n",
    "    salaries = []\n",
    "    features = []\n",
    "    ids = []\n",
    "    areas = []\n",
    "    specializations = []\n",
    "    stemmer = Stemmer.Stemmer('russian')\n",
    "    cursor = db.cursor()\n",
    "    cursor.execute(\"\"\"\n",
    "        SELECT r.item \n",
    "        FROM resumes r \n",
    "        WHERE r.is_active=1 AND NOT EXISTS (\n",
    "            SELECT *\n",
    "            FROM recommendations rec\n",
    "            WHERE rec.resume_id=r.item_id and rec.is_active=1\n",
    "        ) \"\"\")\n",
    "    for item in cursor:\n",
    "        resume_json = json.loads(item[0])\n",
    "        feature = []\n",
    "        #description\n",
    "        p_doc = ''\n",
    "        if resume_json['skills'] != None:\n",
    "            doc = re.sub('<[^>]*>', '', resume_json['skills'].lower())\n",
    "            doc = re.sub('&quot;', '', doc)\n",
    "            doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "            words = re.split(r'\\s{1,}', doc.strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word.strip())\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_doc = p_doc + \" \" + word\n",
    "\n",
    "        #title\n",
    "        p_title = ''\n",
    "        if resume_json['title'] != None:\n",
    "            title = re.sub(ur'[^a-zа-я]+', ' ', resume_json['title'].lower(), re.UNICODE)\n",
    "            words = re.split(r'\\s{1,}', title.strip())\n",
    "            for title_word in words:\n",
    "                title_word = stemmer.stemWord(title_word)\n",
    "                if len(title_word.strip()) > 1:\n",
    "                    p_title = p_title + \" \" + title_word.strip()\n",
    "\n",
    "        #keyskills\n",
    "        p_skills = ''\n",
    "        res_skills = resume_json['skill_set']\n",
    "        for skill in res_skills:\n",
    "            words = re.split(r'\\s{1,}', skill.lower().strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word)\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_skills = p_skills + \" \" + word.strip()\n",
    "\n",
    "        #salary\n",
    "        salary = None\n",
    "        if resume_json['salary'] != None and resume_json['salary']['amount'] != None:\n",
    "            salary = resume_json['salary']['amount']/currency_rates[resume_json['salary']['currency']]\n",
    "        max_salary = 500000.0\n",
    "        if salary >= max_salary:\n",
    "            salary = max_salary\n",
    "            \n",
    "        #experience\n",
    "        if resume_json['experience'] != None and len(resume_json['experience'])> 0 and resume_json['experience'][0]['description'] != None:\n",
    "            experience_description = resume_json['experience'][0]['description']\n",
    "            doc = re.sub('<[^>]*>', '', experience_description.lower())\n",
    "            doc = re.sub('&quot;', '', doc)\n",
    "            doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "            words = re.split(r'\\s{1,}', doc.strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word.strip())\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_doc = p_doc + \" \" + word\n",
    "            \n",
    "        \n",
    "        #areas\n",
    "        res_areas = []\n",
    "        if resume_json['area'] == None:\n",
    "            res_areas.append(areas_map[\"1\"])\n",
    "        else :\n",
    "            res_areas.append(areas_map[resume_json['area']['id']])\n",
    "        for area in resume_json['relocation']['area']:\n",
    "            res_areas.append(areas_map[area['id']])\n",
    "        areas.append(res_areas)\n",
    "        \n",
    "        #specializations\n",
    "        res_specializations = set()\n",
    "        try:\n",
    "            if resume_json['specialization'] != None:\n",
    "                for spec in resume_json['specialization']:\n",
    "                    res_specializations.add(spec['profarea_id'])\n",
    "        except KeyError:\n",
    "            print 'cant find specialization'\n",
    "        specializations.append(res_specializations)\n",
    "               \n",
    "        p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "        feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "        feature = tfidf_transformer.transform(feature_p_doc)\n",
    "        features.append(feature.toarray())\n",
    "        salaries.append(salary)\n",
    "        ids.append(resume_json['id'])\n",
    "    cursor.close()\n",
    "    db.close()\n",
    "    return features, salaries, ids, areas, specializations\n",
    "\n",
    "resume_features, resume_salaries, resume_ids, resume_areas, resume_specializations = get_resumes()\n",
    "lock = threading.Lock()\n",
    "\n",
    "def process_vacancy_ids(vacancies):\n",
    "    pre_vacancy_similarities = {}\n",
    "    pre_vacancy_ids = {}\n",
    "\n",
    "    for idx, val in enumerate(resume_features):\n",
    "        new_vacancy_features = []\n",
    "        new_vacancy_ids = []\n",
    "        new_vacancy_specializations = []\n",
    "        for vac_id, vac_data in vacancies.iteritems():\n",
    "            if resume_areas[idx][0] == vac_data['area'] and (resume_salaries[idx] == None or vac_data['salary'] == 'None'):\n",
    "                new_vacancy_features.append(json.loads(vac_data['features'].decode('zlib')))\n",
    "                new_vacancy_ids.append(vac_id)\n",
    "                if 'specializations' in vac_data:\n",
    "                    new_vacancy_specializations.append(vac_data['specializations'])\n",
    "                else: \n",
    "                    new_vacancy_specializations.append(None)\n",
    "            elif resume_areas[idx][0] == vac_data['area']:\n",
    "                min_resume_salary = resume_salaries[idx] - (resume_salaries[idx] * 0.2)\n",
    "                max_resume_salary = resume_salaries[idx] + (resume_salaries[idx] * 0.8)\n",
    "                vac_salary = float(vac_data['salary'])\n",
    "                if vac_salary >= min_resume_salary and vac_salary <= max_resume_salary:\n",
    "                    new_vacancy_features.append(json.loads(vac_data['features'].decode('zlib')))\n",
    "                    new_vacancy_ids.append(vac_id)\n",
    "                    if 'specializations' in vac_data:\n",
    "                        new_vacancy_specializations.append(vac_data['specializations'])\n",
    "                    else: \n",
    "                        new_vacancy_specializations.append(None)\n",
    "                    \n",
    "        similarities = []\n",
    "        ids = []\n",
    "        if len(new_vacancy_features) > 0:\n",
    "            c_result = cosine_similarity(resume_features[idx], new_vacancy_features)\n",
    "            for s_id, s_val in enumerate(c_result[0]):\n",
    "                if new_vacancy_specializations[s_id] != None:\n",
    "                    found = False\n",
    "                    for vac_spec_id in new_vacancy_specializations[s_id]:\n",
    "                        for res_spec_id in resume_specializations[idx]:\n",
    "                            if vac_spec_id == res_spec_id:\n",
    "                                found = True\n",
    "                                break;\n",
    "                        if found:\n",
    "                            c_result[0][s_id] = c_result[0][s_id] + (c_result[0][s_id]*1.0)\n",
    "                            break;\n",
    "            res = heapq.nlargest(20, range(len(c_result[0])), c_result[0].take)\n",
    "\n",
    "            for j in res:\n",
    "                similarities.append(c_result[0][j])\n",
    "                ids.append(new_vacancy_ids[j])\n",
    "        \n",
    "        lock.acquire()\n",
    "        try:\n",
    "            if resume_ids[idx] not in pre_vacancy_similarities:\n",
    "                pre_vacancy_similarities[resume_ids[idx]] = similarities\n",
    "                pre_vacancy_ids[resume_ids[idx]] = ids\n",
    "            else:\n",
    "                pre_vacancy_similarities[resume_ids[idx]] = pre_vacancy_similarities[resume_ids[idx]] + similarities\n",
    "                pre_vacancy_ids[resume_ids[idx]] = pre_vacancy_ids[resume_ids[idx]] + ids\n",
    "        finally:\n",
    "            lock.release()\n",
    "            \n",
    "    return len(vacancies), pre_vacancy_similarities, pre_vacancy_ids\n",
    "\n",
    "tp_res = [] \n",
    "tpool = Pool(3) \n",
    "def iterate_ids(start):\n",
    "    cnt = 500\n",
    "    rcursor = r.scan(cursor=start, count=cnt)\n",
    "    vacancies = {}\n",
    "    for vac_id in rcursor[1]:\n",
    "        vacancies[vac_id] = r.hgetall(vac_id)\n",
    "    tres = tpool.apply_async(process_vacancy_ids, (vacancies,))\n",
    "    tp_res.append(tres)\n",
    "    while (rcursor[0] != 0):\n",
    "        rcursor = r.scan(cursor=rcursor[0], count=cnt)\n",
    "        vacancies = {}\n",
    "        for vac_id in rcursor[1]:\n",
    "            vacancies[vac_id] = r.hgetall(vac_id)\n",
    "        tres = tpool.apply_async(process_vacancy_ids, (vacancies,))\n",
    "        tp_res.append(tres)\n",
    "\n",
    "iterate_ids(0)\n",
    "\n",
    "c = 0\n",
    "pre_vacancy_similarities = {}\n",
    "pre_vacancy_ids = {}\n",
    "for tr in tp_res:\n",
    "    cnt, p_vacancy_similarities, p_vacancy_ids = tr.get()\n",
    "    for resume_id in p_vacancy_similarities.keys():\n",
    "        if resume_id not in pre_vacancy_similarities:\n",
    "            pre_vacancy_similarities[resume_id] = p_vacancy_similarities[resume_id]\n",
    "            pre_vacancy_ids[resume_id] = p_vacancy_ids[resume_id]\n",
    "        else:\n",
    "            pre_vacancy_similarities[resume_id] = pre_vacancy_similarities[resume_id]+p_vacancy_similarities[resume_id]\n",
    "            pre_vacancy_ids[resume_id] = pre_vacancy_ids[resume_id]+p_vacancy_ids[resume_id]\n",
    "    \n",
    "    c = c+cnt\n",
    "    print 'processed {}'.format(c)\n",
    "\n",
    "def finalize_recommendations(resume_id):\n",
    "    result = []\n",
    "    similarities = pre_vacancy_similarities[resume_id]\n",
    "    ids = pre_vacancy_ids[resume_id]\n",
    "    max_similarities = heapq.nlargest(20, range(len(numpy.asarray(similarities))), numpy.asarray(similarities).take)\n",
    "\n",
    "    db = MySQLdb.connect(host=\"127.0.0.1\", \n",
    "                     port=config.getint('mysqld', 'port'), \n",
    "                     user=config.get('mysqld', 'user'), \n",
    "                     passwd=config.get('mysqld', 'password'), \n",
    "                     db=config.get('mysqld', 'database') )\n",
    "    db.autocommit(True)\n",
    "    db.set_character_set('utf8')\n",
    "    cursor = db.cursor()\n",
    "    cursor.execute('SET NAMES utf8;')\n",
    "    cursor.execute('SET CHARACTER SET utf8;')\n",
    "    cursor.execute('SET character_set_connection=utf8;')\n",
    "    cursor.close()\n",
    "\n",
    "    cursor = db.cursor()\n",
    "    try:\n",
    "        cursor.execute(\"\"\"UPDATE recommendations SET is_active=0 WHERE resume_id='{}'\"\"\".format(resume_id))\n",
    "    except BaseException as ex:\n",
    "        print ex\n",
    "    finally:\n",
    "        cursor.close()\n",
    "\n",
    "        \n",
    "    for ind in max_similarities:\n",
    "        conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "        conn.request(\"GET\", \"https://api.hh.ru/vacancies/{}\".format(ids[ind]), headers=headers)\n",
    "        r1 = conn.getresponse()\n",
    "        if r1.status != 200:\n",
    "            conn.close()\n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/vacancies/{}\".format(ids[ind]), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "        t_vacancy = r1.read()\n",
    "        conn.close()\n",
    "        t_vacancy_json = json.loads(t_vacancy)\n",
    "        try:\n",
    "            title = t_vacancy_json['name'].encode('utf-8').strip()\n",
    "        except KeyError as ex:\n",
    "            print ex\n",
    "            title = 'Title temporary not found'\n",
    "\n",
    "        \n",
    "        cursor = db.cursor()\n",
    "        try:\n",
    "            cursor.execute(\"\"\"\n",
    "                INSERT INTO recommendations (resume_id, vacancy_id, updated, is_active, similarity, vacancy_title) \n",
    "                VALUES ('{}', {}, now(), 1, {}, '{}')\n",
    "            \"\"\".format(resume_id, ids[ind], similarities[ind], title))\n",
    "        except BaseException as err:\n",
    "            print err\n",
    "        finally:\n",
    "            cursor.close()\n",
    "        \n",
    "        result.append('{}. for {} similarity is {}'.format(resume_id, ids[ind], similarities[ind]))\n",
    "\n",
    "    db.close()\n",
    "    return result\n",
    "\n",
    "p_res = [] \n",
    "pool = Pool(7) \n",
    "for resume_id in pre_vacancy_similarities.keys():\n",
    "    res = pool.apply_async(finalize_recommendations, (resume_id,))\n",
    "    p_res.append(res)\n",
    "    \n",
    "for t in p_res:\n",
    "    res = t.get()\n",
    "    for s in res:\n",
    "        print s\n",
    "        \n",
    "\n",
    "print 'total time {} sec\\n'.format(time.time()-start_time)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "абр\n",
      "хабр\n"
     ]
    }
   ],
   "source": [
    "import Stemmer\n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "print stemmer.stemWord('абром')\n",
    "print stemmer.stemWord('хабру')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start at 2016-06-22 15:07:56.115747\n",
      "/vacancies?per_page=200&date_from=2016-06-22T15:02:56&date_to=2016-06-22T15:07:56&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T15:02:56&date_to=2016-06-22T15:07:56&page=1\n",
      "310\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:58:36&date_to=2016-06-22T15:03:36&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:58:36&date_to=2016-06-22T15:03:36&page=1\n",
      "310\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:54:17&date_to=2016-06-22T14:59:17&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:54:17&date_to=2016-06-22T14:59:17&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:54:17&date_to=2016-06-22T14:59:17&page=2\n",
      "421\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:50:05&date_to=2016-06-22T14:55:05&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:50:05&date_to=2016-06-22T14:55:05&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:50:05&date_to=2016-06-22T14:55:05&page=2\n",
      "421\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:45:53&date_to=2016-06-22T14:50:53&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:45:53&date_to=2016-06-22T14:50:53&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:45:53&date_to=2016-06-22T14:50:53&page=2\n",
      "509\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "starting t6\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:41:46&date_to=2016-06-22T14:46:46&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:41:46&date_to=2016-06-22T14:46:46&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:41:46&date_to=2016-06-22T14:46:46&page=2\n",
      "451\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:37:36&date_to=2016-06-22T14:42:36&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:37:36&date_to=2016-06-22T14:42:36&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:37:36&date_to=2016-06-22T14:42:36&page=2\n",
      "426\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:33:23&date_to=2016-06-22T14:38:23&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:33:23&date_to=2016-06-22T14:38:23&page=1\n",
      "341\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:29:05&date_to=2016-06-22T14:34:05&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:29:05&date_to=2016-06-22T14:34:05&page=1\n",
      "269\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:24:43&date_to=2016-06-22T14:29:43&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:24:43&date_to=2016-06-22T14:29:43&page=1\n",
      "372\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:20:27&date_to=2016-06-22T14:25:27&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:20:27&date_to=2016-06-22T14:25:27&page=1\n",
      "372\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:16:11&date_to=2016-06-22T14:21:11&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:16:11&date_to=2016-06-22T14:21:11&page=1\n",
      "385\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:11:55&date_to=2016-06-22T14:16:55&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:11:55&date_to=2016-06-22T14:16:55&page=1\n",
      "353\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:07:37&date_to=2016-06-22T14:12:37&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:07:37&date_to=2016-06-22T14:12:37&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:07:37&date_to=2016-06-22T14:12:37&page=2\n",
      "410\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:03:28&date_to=2016-06-22T14:08:28&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T14:03:28&date_to=2016-06-22T14:08:28&page=1\n",
      "371\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:59:10&date_to=2016-06-22T14:04:10&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:59:10&date_to=2016-06-22T14:04:10&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:59:10&date_to=2016-06-22T14:04:10&page=2\n",
      "401\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:54:56&date_to=2016-06-22T13:59:56&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:54:56&date_to=2016-06-22T13:59:56&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:54:56&date_to=2016-06-22T13:59:56&page=2\n",
      "469\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:50:46&date_to=2016-06-22T13:55:46&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:50:46&date_to=2016-06-22T13:55:46&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:50:46&date_to=2016-06-22T13:55:46&page=2\n",
      "469\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:46:35&date_to=2016-06-22T13:51:35&page=0\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:46:35&date_to=2016-06-22T13:51:35&page=1\n",
      "/vacancies?per_page=200&date_from=2016-06-22T13:46:35&date_to=2016-06-22T13:51:35&page=2\n",
      "482\n",
      "starting t1\n",
      "starting t2\n",
      "starting t3\n",
      "starting t4\n",
      "starting t5\n",
      "873.098965883 sec\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "#download many vacancies\n",
    "import httplib\n",
    "import json\n",
    "import time\n",
    "import threading\n",
    "import pickle\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import heapq\n",
    "import numpy\n",
    "import re \n",
    "import Stemmer\n",
    "import datetime\n",
    "import redis\n",
    "\n",
    "\n",
    "print 'Start at {}'.format(datetime.datetime.now())\n",
    "r = redis.StrictRedis(host='localhost', port=6379, db=0)\n",
    "start_time = time.time()\n",
    "timeout = 5*24*60*60\n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "if r1.status != 200:\n",
    "    conn.close()\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    conn.request(\"GET\", \"https://api.hh.ru/dictionaries\", headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "dictionaries = r1.read()\n",
    "dictionaries_json = json.loads(dictionaries)\n",
    "currencies = dictionaries_json['currency']\n",
    "currency_rates = {}\n",
    "for currency in currencies:\n",
    "    currency_rates[currency['code']] = currency['rate']\n",
    "    \n",
    "#areas\n",
    "conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "conn.request(\"GET\", \"https://api.hh.ru/areas\", headers=headers)\n",
    "r1 = conn.getresponse()\n",
    "if r1.status != 200:\n",
    "    conn.close()\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    conn.request(\"GET\", \"https://api.hh.ru/areas\", headers=headers)\n",
    "    r1 = conn.getresponse()\n",
    "areas = r1.read()\n",
    "conn.close()\n",
    "areas_json = json.loads(areas)\n",
    "areas_map = {}\n",
    "def build_areas_map(areas, areas_map):\n",
    "    for area in areas:\n",
    "        if area['id'] == '1':#msk\n",
    "            parent_id = '2019'\n",
    "        elif area['id'] == '2':#spb\n",
    "            parent_id = '145'\n",
    "        elif area['id'] == '115':#kiev\n",
    "            parent_id = '2164'\n",
    "        elif area['id'] == '1002':#minsk\n",
    "            parent_id = '2237'\n",
    "        else:\n",
    "            parent_id = area['parent_id']\n",
    "        areas_map[area['id']] = parent_id\n",
    "        build_areas_map(area['areas'], areas_map)\n",
    "        \n",
    "build_areas_map(areas_json, areas_map)\n",
    "    \n",
    "with open( \"count_vectorizer.p\", \"rb\" ) as f:\n",
    "    count_vectorizer = pickle.load(f)\n",
    "    \n",
    "with open( \"tfidf_transformer.p\", \"rb\" ) as f:\n",
    "    tfidf_transformer = pickle.load(f)\n",
    "    \n",
    "stemmer = Stemmer.Stemmer('russian')\n",
    "    \n",
    "headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "\n",
    "def get_vacancy_ids(cnt):\n",
    "    vacancy_ids = []\n",
    "    conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "    per_page = 200\n",
    "    page = 0\n",
    "    count = per_page\n",
    "    date_from = (datetime.datetime.now() - datetime.timedelta(minutes=((cnt-1)*5)+5)).strftime('%Y-%m-%dT%H:%M:%S')\n",
    "    date_to = (datetime.datetime.now() - datetime.timedelta(minutes=((cnt-1)*5))).strftime('%Y-%m-%dT%H:%M:%S')\n",
    "    while count == per_page:\n",
    "        path = (\"/vacancies?per_page={}&date_from={}&date_to={}&page={}\"\n",
    "                .format(per_page, date_from, date_to, page))\n",
    "        print path\n",
    "\n",
    "        conn.request(\"GET\", path, headers=headers)\n",
    "        r1 = conn.getresponse()\n",
    "        vacancies = r1.read()\n",
    "        conn.close()\n",
    "\n",
    "        count = len(json.loads(vacancies)['items'])\n",
    "        page = page+1\n",
    "        for item in json.loads(vacancies)['items']:\n",
    "            vacancy_ids.append(item['id'])\n",
    "    return vacancy_ids\n",
    "        \n",
    "\n",
    "def process_vacancies(vacancy_ids):\n",
    "    headers = {\"User-Agent\": \"hh-recommender\"}\n",
    "    for vac_id in vacancy_ids:\n",
    "        conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "        conn.request(\"GET\", \"/vacancies/{}\".format(vac_id), headers=headers)\n",
    "        r1 = conn.getresponse()\n",
    "        if r1 != 200:\n",
    "            conn.close()\n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"/vacancies/{}\".format(vac_id), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "        vacancy_txt = r1.read()\n",
    "        conn.close()\n",
    "        vacancy = json.loads(vacancy_txt)\n",
    "        \n",
    "        feature = []\n",
    "\n",
    "        #description\n",
    "        p_doc = ''\n",
    "        doc = re.sub('<[^>]*>', '', vacancy['description'].lower())\n",
    "        doc = re.sub('&quot;', '', doc)\n",
    "        doc = re.sub(ur'[^a-zа-я]+', ' ', doc, re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', doc.strip())\n",
    "        for word in words:\n",
    "            word = stemmer.stemWord(word.strip())\n",
    "            if len(word.strip()) > 1:\n",
    "                p_doc = p_doc + \" \" + word\n",
    "\n",
    "        #title\n",
    "        p_title = ''\n",
    "        title = re.sub(ur'[^a-zа-я]+', ' ', vacancy['name'].lower(), re.UNICODE)\n",
    "        words = re.split(r'\\s{1,}', title.strip())\n",
    "        for title_word in words:\n",
    "            title_word = stemmer.stemWord(title_word)\n",
    "            if len(title_word.strip()) > 1:\n",
    "                p_title = p_title + \" \" + title_word.strip()\n",
    "\n",
    "        #keyskills\n",
    "        p_skills = ''\n",
    "        vac_skills = vacancy['key_skills']\n",
    "        for skill in vac_skills:\n",
    "            words = re.split(r'\\s{1,}', skill['name'].lower().strip())\n",
    "            for word in words:\n",
    "                word = stemmer.stemWord(word)\n",
    "                if len(word.strip()) > 1:\n",
    "                    p_skills = p_skills + \" \" + word.strip()\n",
    "                    \n",
    "        #salary\n",
    "        salary = None\n",
    "        if vacancy['salary'] != None:\n",
    "            if vacancy['salary']['from'] == None and vacancy['salary']['to'] != None:\n",
    "                salary = vacancy['salary']['to']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] == None and vacancy['salary']['from'] != None:\n",
    "                salary = vacancy['salary']['from']/currency_rates[vacancy['salary']['currency']]\n",
    "            elif vacancy['salary']['to'] != None and vacancy['salary']['from'] != None:\n",
    "                salary = ((vacancy['salary']['from'] + vacancy['salary']['to'])/2)/currency_rates[vacancy['salary']['currency']]\n",
    "        max_salary = 500000.0\n",
    "        if salary >= max_salary:\n",
    "            salary = max_salary\n",
    "\n",
    "        try:\n",
    "            area_id = areas_map[vacancy['area']['id']]\n",
    "        except KeyError:\n",
    "            print 'missed area id {} for vacancy {}'.format(vacancy['area']['id'], vacancy['id'])\n",
    "            \n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/vacancies/{}\".format(vacancy['id']), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "            if r1 != 200:\n",
    "                conn.close()\n",
    "                conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "                conn.request(\"GET\", \"https://api.hh.ru/vacancies/{}\".format(vacancy['id']), headers=headers)\n",
    "                r1 = conn.getresponse()\n",
    "            missed_area_vacancy = r1.read()\n",
    "            conn.close()\n",
    "            missed_area_vacancy_json = json.loads(missed_area_vacancy)\n",
    "            \n",
    "            conn = httplib.HTTPSConnection(\"api.hh.ru\")\n",
    "            conn.request(\"GET\", \"https://api.hh.ru/areas/{}\"\n",
    "                         .format(missed_area_vacancy_json['area']['id']), headers=headers)\n",
    "            r1 = conn.getresponse()\n",
    "            missed_area = r1.read()\n",
    "            conn.close()\n",
    "            missed_area_json = json.loads(missed_area)\n",
    "            areas_map[vacancy['area']['id']] = missed_area_json['parent_id']\n",
    "            area_id = missed_area_json['parent_id']\n",
    "\n",
    "        p_doc = p_doc + \" \" + p_title + \" \" + p_skills\n",
    "        \n",
    "        #specializations\n",
    "        specializations = set()\n",
    "        try:\n",
    "            if vacancy['specializations'] != None:\n",
    "                for spec in vacancy['specializations']:\n",
    "                    specializations.add(spec['profarea_id'])\n",
    "        except KeyError:\n",
    "            print 'cant find specialization'\n",
    "\n",
    "        feature_p_doc = count_vectorizer.transform([p_doc])\n",
    "        tfidf_feature_p_doc = tfidf_transformer.transform(feature_p_doc)\n",
    "            \n",
    "        data = {}\n",
    "        data['features'] = json.dumps(tfidf_feature_p_doc.toarray()[0].tolist()).encode(\"zlib\")\n",
    "        data['salary'] = salary\n",
    "        data['area'] = area_id\n",
    "        data['specializations'] = specializations\n",
    "        r.hmset(vacancy['id'], data)\n",
    "        r.expire(vacancy['id'], timeout)\n",
    "\n",
    "for num in range(1,600):\n",
    "    print 'start interval {}'.format(num)\n",
    "    ids = get_vacancy_ids(num)\n",
    "    print len(ids)\n",
    "    vac_id_chunks=[ids[x:x+100] for x in xrange(0, len(ids), 100)]\n",
    "    t_num = 1;\n",
    "    threads = []\n",
    "    for vac_id_chunk in vac_id_chunks:\n",
    "        print 'starting t{}'.format(t_num)\n",
    "        t_num = t_num + 1\n",
    "        t = threading.Thread(target=process_vacancies, kwargs={'vacancy_ids': vac_id_chunk})\n",
    "        threads.append(t)\n",
    "        t.start()\n",
    "\n",
    "    for t in threads:\n",
    "        t.join()\n",
    "    \n",
    "\n",
    "print \"{} sec\".format(time.time() - start_time)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}