Adding demo spider

Pipfile

@@ -43,6 +43,10 @@ cufflinks = "*"
 tables = "*"
 nb-black = "*"
 mypy = "*"
+scrapy = "*"
+slugify = "*"
+shub = "*"
+pipenv = "*"
 
 [requires]
 python_version = "3.7"

src/demo_spiders/Pipfile

@@ -0,0 +1,27 @@
+[[source]]
+url = "https://pypi.python.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+jmespath = "*"
+pudb = "*"
+scrapy = "==1.6"
+python-slugify = "*"
+
+[dev-packages]
+Scrapy = ">=1.6.0,<1.7.0"
+pytest = "*"
+pytest-twisted = "*"
+pytest-cov = "*"
+mock = "*"
+flake8 = "*"
+flake8-bugbear = "*"
+shub = "*"
+Sphinx = "*"
+pipenv = "*"
+pylint = "*"
+rope = "*"
+
+[requires]
+python_version = "3.7"

src/demo_spiders/build/lib/demo_spiders/items.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your scraped items
+#
+# See documentation in:
+# https://doc.scrapy.org/en/latest/topics/items.html
+import re
+from enum import Enum
+
+from scrapy import Item, Field
+from scrapy.selector import Selector, SelectorList
+from scrapy.loader import ItemLoader
+from scrapy.loader.processors import TakeFirst, Compose, Identity
+from slugify import slugify
+
+from .processors import clean_output, only_alphanumeric, get_money, boolean_output
+
+
+class Availability(Enum):
+    UNKNOWN = 0
+    IN_STOCK = 1
+    OUT_OF_STOCK = 2
+    IN_STORE_ONLY = 3
+    BACKORDERED = 4
+    SHIP_FROM_STORE = 5
+    LIMITED_DELIVERY = 6
+
+    @classmethod
+    def from_string(cls, value):
+        value = value.lower().strip()
+        value = slugify(value, separator='_')
+        if value in {'instock', 'in_stock', 'limited_stock', 'onlineonly', 'available', 'lowinstock'}:
+            return cls.IN_STOCK
+        elif value in {'discontinued', 'outofstock', 'out_of_stock', 'soldout', 'unavailable'}:
+            return cls.OUT_OF_STOCK
+        elif value in {'instoreonly'}:
+            return cls.IN_STORE_ONLY
+        elif value in {'limitedavailability'}:
+            return cls.LIMITED_DELIVERY
+        else:
+            return cls.UNKNOWN
+
+
+class ProductItem(Item):
+    url = Field()
+    name = Field()
+    part_number = Field()
+    availability = Field()
+    price = Field()
+    list_price = Field()
+    category = Field()
+    thirdparty = Field()
+    seller = Field()
+    rating = Field()
+    rating_count = Field()
+
+    subcategory = Field()
+    quantity = Field()
+    uom = Field()
+    coupon = Field()
+    subscription_price = Field()
+    altpartnumber = Field()
+    gtin = Field()
+    manufacturer = Field()
+    mfpartnumber = Field()
+    qtylimit = Field()
+    min_purchase_quantity = Field()
+
+
+class ProductItemLoader(ItemLoader):
+    default_item_class = ProductItem
+    default_input_processor = Identity()
+    default_output_processor = Compose(TakeFirst(), clean_output)
+
+    name_out = Compose(default_output_processor, only_alphanumeric)
+    seller_out = Compose(default_output_processor, only_alphanumeric)
+    rating_out = Compose(default_output_processor, float)
+    rating_count_out = Compose(default_output_processor, int)
+    quantity_out = Compose(default_output_processor, float)
+    qtylimit_out = Compose(default_output_processor, int)
+    min_purchase_quantity_out = Compose(default_output_processor, int)
+    thirdparty_out = Compose(TakeFirst(), boolean_output)
+    availability_out = Compose(TakeFirst(), lambda a: a.value if a else None)
+    price_out = Compose(default_output_processor, get_money)
+    list_price_out = Compose(default_output_processor, get_money)
+    subscription_price_out = Compose(default_output_processor, get_money)
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        response = kwargs.get('response')
+        if not response:
+            return
+
+        # self.add_value('url', response.url)
+
+    def add_value(self, field, value, *args, **kwargs):
+        if isinstance(value, (Selector, SelectorList)):
+            value = value.extract()
+        super().add_value(field, value, *args, **kwargs)
+
+    @classmethod
+    def apply_regex(cls, pattern, value):
+        match = re.findall(pattern, value)
+        if match:
+            return match[0]
+        raise ValueError(f'Could not parse rating in "{value}"')
+
+
+class DiscoveryItem(Item):
+    url = Field()
+
+
+class BooksItem(Item):
+    title = Field()
+    category = Field()
+    price = Field()
+    description = Field()
+
+
+class BooksItemLoader(ItemLoader):
+    default_item_class = BooksItem
+    default_input_processor = Identity()

src/demo_spiders/build/lib/demo_spiders/middlewares.py

@@ -0,0 +1,103 @@
+# -*- coding: utf-8 -*-
+
+# Define here the models for your spider middleware
+#
+# See documentation in:
+# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
+
+from scrapy import signals
+
+
+class DemoSpidersSpiderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the spider middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_spider_input(self, response, spider):
+        # Called for each response that goes through the spider
+        # middleware and into the spider.
+
+        # Should return None or raise an exception.
+        return None
+
+    def process_spider_output(self, response, result, spider):
+        # Called with the results returned from the Spider, after
+        # it has processed the response.
+
+        # Must return an iterable of Request, dict or Item objects.
+        for i in result:
+            yield i
+
+    def process_spider_exception(self, response, exception, spider):
+        # Called when a spider or process_spider_input() method
+        # (from other spider middleware) raises an exception.
+
+        # Should return either None or an iterable of Request, dict
+        # or Item objects.
+        pass
+
+    def process_start_requests(self, start_requests, spider):
+        # Called with the start requests of the spider, and works
+        # similarly to the process_spider_output() method, except
+        # that it doesn’t have a response associated.
+
+        # Must return only requests (not items).
+        for r in start_requests:
+            yield r
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)
+
+
+class DemoSpidersDownloaderMiddleware(object):
+    # Not all methods need to be defined. If a method is not defined,
+    # scrapy acts as if the downloader middleware does not modify the
+    # passed objects.
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        # This method is used by Scrapy to create your spiders.
+        s = cls()
+        crawler.signals.connect(s.spider_opened, signal=signals.spider_opened)
+        return s
+
+    def process_request(self, request, spider):
+        # Called for each request that goes through the downloader
+        # middleware.
+
+        # Must either:
+        # - return None: continue processing this request
+        # - or return a Response object
+        # - or return a Request object
+        # - or raise IgnoreRequest: process_exception() methods of
+        #   installed downloader middleware will be called
+        return None
+
+    def process_response(self, request, response, spider):
+        # Called with the response returned from the downloader.
+
+        # Must either;
+        # - return a Response object
+        # - return a Request object
+        # - or raise IgnoreRequest
+        return response
+
+    def process_exception(self, request, exception, spider):
+        # Called when a download handler or a process_request()
+        # (from other downloader middleware) raises an exception.
+
+        # Must either:
+        # - return None: continue processing this exception
+        # - return a Response object: stops process_exception() chain
+        # - return a Request object: stops process_exception() chain
+        pass
+
+    def spider_opened(self, spider):
+        spider.logger.info('Spider opened: %s' % spider.name)

src/demo_spiders/build/lib/demo_spiders/pipelines.py

@@ -0,0 +1,11 @@
+# -*- coding: utf-8 -*-
+
+# Define your item pipelines here
+#
+# Don't forget to add your pipeline to the ITEM_PIPELINES setting
+# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
+
+
+class DemoSpidersPipeline(object):
+    def process_item(self, item, spider):
+        return item