add dianping

dianping/dianping/items.py

@@ -0,0 +1,11 @@
+# Define here the models for your scraped items
+#
+# See documentation in:
+# http://doc.scrapy.org/en/latest/topics/items.html
+
+from scrapy.item import Item, Field
+
+class dianpingItem(Item):
+    # define the fields for your item here like:
+    name = Field()
+

dianping/dianping/pipelines.py

@@ -0,0 +1,50 @@
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
+
+import redis
+
+
+from scrapy import signals
+
+
+import json
+import codecs
+from collections import OrderedDict
+
+
+class JsonWithEncodingPipeline(object):
+
+    def __init__(self):
+        self.file = codecs.open('data_utf8.json', 'w', encoding='utf-8')
+
+    def process_item(self, item, spider):
+        line = json.dumps(OrderedDict(item), ensure_ascii=False, sort_keys=False) + "\n"
+        self.file.write(line)
+        return item
+
+    def close_spider(self, spider):
+        self.file.close()
+
+
+class RedisPipeline(object):
+
+    def __init__(self):
+        self.r = redis.StrictRedis(host='localhost', port=6379)
+
+    def process_item(self, item, spider):
+        if not item['id']:
+            print 'no id item!!'
+
+        str_recorded_item = self.r.get(item['id'])
+        final_item = None
+        if str_recorded_item is None:
+            final_item = item
+        else:
+            ritem = eval(self.r.get(item['id']))
+            final_item = dict(item.items() + ritem.items())
+        self.r.set(item['id'], final_item)
+
+    def close_spider(self, spider):
+        return

dianping/dianping/settings.py

@@ -0,0 +1,36 @@
+# Scrapy settings for dianping project
+#
+# For simplicity, this file contains only the most important settings by
+# default. All the other settings are documented here:
+#
+#     http://doc.scrapy.org/en/latest/topics/settings.html
+#
+
+import sys
+import os
+from os.path import dirname
+path = dirname(dirname(os.path.abspath(os.path.dirname(__file__))))
+sys.path.append(path)
+from misc.log import *
+
+BOT_NAME = 'dianping'
+
+SPIDER_MODULES = ['dianping.spiders']
+NEWSPIDER_MODULE = 'dianping.spiders'
+
+# Crawl responsibly by identifying yourself (and your website) on the user-agent
+#USER_AGENT = 'dianping (+http://www.yourdomain.com)'
+
+DOWNLOADER_MIDDLEWARES = {
+   # 'misc.middleware.CustomHttpProxyMiddleware': 400,
+    'misc.middleware.CustomUserAgentMiddleware': 401,
+}
+
+ITEM_PIPELINES = {
+    'dianping.pipelines.JsonWithEncodingPipeline': 300,
+    #'dianping.pipelines.RedisPipeline': 301,
+}
+
+LOG_LEVEL = 'INFO'
+
+DOWNLOAD_DELAY = 1

dianping/dianping/spiders/__init__.py

@@ -0,0 +1,4 @@
+# This package will contain the spiders of your Scrapy project
+#
+# Please refer to the documentation for information on how to create and manage
+# your spiders.

dianping/dianping/spiders/spider.py

@@ -0,0 +1,36 @@
+from scrapy.http import Request
+from scrapy.selector import Selector
+
+try:
+    from scrapy.spiders import Spider
+except:
+    from scrapy.spiders import BaseSpider as Spider
+
+from misc.spider import CommonSpider
+
+
+class dianpingSpider(CommonSpider):
+    name = "dianping"
+    allowed_domains = ["dianping.com"]
+    start_urls = [
+        "http://www.dianping.com/search/category/2/30/g141r1471",
+    ]
+
+    def parse(self, response):
+        hxs = Selector(response)
+        sites = hxs.xpath('//div[@class="tit"]/a/@href').extract()
+        for site in sites:
+            if site.startswith('/shop/'):
+                yield Request("http://www.dianping.com{}".format(site), callback=self.parse_shop)
+
+    def parse_shop(self, response):
+        pass
+
+        # categories = hxs.xpath(
+        #     '//div[@class="%s"]/following-sibling::ul/li[@class="item has-panel"]' % self.MEDIA_CLASSES[
+        #         self.media_type])
+        # for category in categories:
+        #     name = category.xpath('a/text()').extract()[0].strip()
+        #     if name in self.IGNORE_CATEGORIES[self.media_type]:
+        #         continue
+        #     source_url = category.xpath('a/@href').extract()[0].strip()

dianping/scrapy.cfg

@@ -0,0 +1,11 @@
+# Automatically created by: scrapy startproject
+#
+# For more information about the [deploy] section see:
+# http://doc.scrapy.org/en/latest/topics/scrapyd.html
+
+[settings]
+default = dianping.settings
+
+[deploy]
+#url = http://localhost:6800/
+project = dianping