main.py

import datetime

from scrapy.selector import Selector
import matplotlib.pyplot as plt
import aprox
from spider import Jumper

gks_ru = Jumper(url='http://www.gks.ru/dbscripts/cbsd/dbinet.cgi', proxies=None)
data = 'rdLayoutType=Au&_Pokazateli=on&_okato=on&_grtov=on&_god=on&_period=on&a_Pokazateli=1&a_okato=2&a_grtov=3&Qry=Pokazateli%3A1921002%3Bokato%3A80000000%3Bgrtov%3A1501%3Bgod%3A2011%2C2012%2C2013%2C2014%2C2015%3Bperiod%3A12011%2C22011%2C32011%2C42011%2C52011%2C62011%2C72011%2C82011%2C92011%2C102011%2C112011%2C122011%2C132011%2C142011%2C152011%2C162011%2C172011%2C182011%2C192011%2C202011%2C212011%2C222011%2C232011%2C242011%2C252011%2C262011%2C272011%2C282011%2C292011%2C302011%2C312011%2C322011%2C332011%2C342011%2C352011%2C362011%2C372011%2C382011%2C392011%2C402011%2C412011%2C422011%2C432011%2C442011%2C452011%2C462011%2C472011%2C482011%2C492011%2C502011%2C512011%2C12012%2C22012%2C32012%2C42012%2C52012%2C62012%2C72012%2C82012%2C92012%2C102012%2C112012%2C122012%2C132012%2C142012%2C152012%2C162012%2C172012%2C182012%2C192012%2C202012%2C212012%2C222012%2C232012%2C242012%2C252012%2C262012%2C272012%2C282012%2C292012%2C302012%2C312012%2C322012%2C332012%2C342012%2C352012%2C362012%2C372012%2C382012%2C392012%2C402012%2C412012%2C422012%2C432012%2C442012%2C452012%2C462012%2C472012%2C482012%2C492012%2C502012%2C512012%2C12013%2C22013%2C32013%2C42013%2C52013%2C62013%2C72013%2C82013%2C92013%2C102013%2C112013%2C122013%2C132013%2C142013%2C152013%2C162013%2C172013%2C182013%2C192013%2C202013%2C212013%2C222013%2C232013%2C242013%2C252013%2C262013%2C272013%2C282013%2C292013%2C302013%2C312013%2C322013%2C332013%2C342013%2C352013%2C362013%2C372013%2C382013%2C392013%2C402013%2C412013%2C422013%2C432013%2C442013%2C452013%2C462013%2C472013%2C482013%2C492013%2C502013%2C512013%2C522013%2C12014%2C22014%2C32014%2C42014%2C52014%2C62014%2C72014%2C82014%2C92014%2C102014%2C112014%2C122014%2C132014%2C142014%2C152014%2C162014%2C172014%2C182014%2C192014%2C202014%2C212014%2C222014%2C232014%2C242014%2C252014%2C262014%2C272014%2C282014%2C292014%2C302014%2C312014%2C322014%2C332014%2C342014%2C352014%2C362014%2C372014%2C382014%2C392014%2C402014%2C412014%2C422014%2C432014%2C442014%2C452014%2C462014%2C472014%2C482014%2C492014%2C502014%2C512014%2C12015%2C22015%2C32015%2C42015%2C52015%2C62015%2C72015%2C82015%2C92015%2C102015%2C112015%2C122015%2C132015%2C142015%2C152015%2C162015%2C172015%2C182015%2C192015%2C202015%2C212015%2C222015%2C232015%2C242015%2C252015%2C262015%2C272015%2C282015%2C292015%2C302015%2C312015%2C322015%2C332015%2C342015%2C352015%2C362015%2C372015%2C382015%2C392015%2C402015%2C412015%2C422015%2C432015%2C442015%2C452015%2C462015%2C472015%2C482015%3B&QryGm=Pokazateli_z%3A1%3Bokato_z%3A2%3Bgrtov_z%3A3%3Bgod_s%3A1%3Bperiod_b%3A1%3B&QryFootNotes=%3B&YearsList=2011%3B2012%3B2013%3B2014%3B2015%3B&tbl=%CF%EE%EA%E0%E7%E0%F2%FC+%F2%E0%E1%EB%E8%F6%F3'
content = gks_ru.post(None,data,True)
#file = open('content.html')
#content = file.read()
content = content.decode('cp1251')
content = content.replace(u",", u'.')

xpath = '//table[@class="OutTbl"]/tr[1]/td/text()'
years = [int(td.extract()) for td in Selector(text=content).xpath(xpath)]
values = []
weeks = []
print years
xpath = '//table[@class="OutTbl"]/tr'
count = 0
param = 2
column = 0
for tr in Selector(text=content).xpath(xpath)[1:]:
    count += 1
    xpath = u'//td[{}]/text()'.format(param)
    tdn = Selector(text=tr.extract()).xpath(xpath).extract()[0]
    if tdn != u'\xa0':
        values.append(float(tdn))
    else:
        param += 1
        xpath = u'//td[{}]/text()'.format(param)
        tdn = Selector(text=tr.extract()).xpath(xpath).extract()[0]
        values.append(float(tdn))

dt1 = datetime.date(years[0], 1, 1)
dt2 = dt1
dates = []
for i in range(0, len(values)):
    dt2 = dt2 + datetime.timedelta(days=7)
    dates.append(dt2)

apr=aprox.compute(values)
y1 = apr[1]
y2 = (len(values)*apr[0])+apr[1]

plt.plot(dates, values)
plt.plot([dt1,dt2],[y1,y2])
plt.gcf().autofmt_xdate()
plt.show()