Merge branch 'dev' into ui

Author: Madison Bahmer

.gitignore

@@ -1,6 +1,10 @@
+.vscode
+
 # Python binaries
 *.pyc
 
+.idea/
+
 # Sphinx
 docs/_build
 docs/_build_html

.travis.yml

@@ -20,6 +20,9 @@ env:
   - docker: 1
     dockerfile_name: Dockerfile.py2alpine
     docker_tag_suffix: dev-alpine
+  - docker: 1
+    dockerfile_name: Dockerfile.py3
+    docker_tag_suffix: dev-py3
 
 install: true
 

README.md

@@ -14,7 +14,7 @@ Please see the ``requirements.txt`` within each sub project for Pip package depe
 
 Other important components required to run the cluster
 
-- Python 2.7: https://www.python.org/downloads/
+- Python 2.7 or 3.6: https://www.python.org/downloads/
 - Redis: http://redis.io
 - Zookeeper: https://zookeeper.apache.org
 - Kafka: http://kafka.apache.org

crawler/crawling/distributed_scheduler.py

@@ -282,7 +282,7 @@ def update_ipaddress(self):
         try:
             obj = urllib.request.urlopen(settings.get('PUBLIC_IP_URL',
                                   'http://ip.42.pl/raw'))
-            results = self.ip_regex.findall(obj.read())
+            results = self.ip_regex.findall(obj.read().decode('utf-8'))
             if len(results) > 0:
                 self.my_ip = results[0]
             else:
@@ -313,7 +313,8 @@ def report_self(self):
     def from_settings(cls, settings):
         server = redis.Redis(host=settings.get('REDIS_HOST'),
                              port=settings.get('REDIS_PORT'),
-                             db=settings.get('REDIS_DB'))
+                             db=settings.get('REDIS_DB'),
+                             decode_responses=True)
         persist = settings.get('SCHEDULER_PERSIST', True)
         up_int = settings.get('SCHEDULER_QUEUE_REFRESH', 10)
         hits = settings.get('QUEUE_HITS', 10)
@@ -501,7 +502,7 @@ def next_request(self):
 
         item = self.find_item()
         if item:
-            self.logger.debug("Found url to crawl {url}" \
+            self.logger.debug(u"Found url to crawl {url}" \
                     .format(url=item['url']))
             try:
                 req = Request(item['url'])

crawler/crawling/log_retry_middleware.py

@@ -61,7 +61,8 @@ def setup(self, settings):
         if self.settings['STATS_STATUS_CODES']:
             self.redis_conn = redis.Redis(host=self.settings.get('REDIS_HOST'),
                                           port=self.settings.get('REDIS_PORT'),
-                                          db=settings.get('REDIS_DB'))
+                                          db=settings.get('REDIS_DB'),
+                                          decode_responses=True)
 
             try:
                 self.redis_conn.info()

crawler/crawling/pipelines.py

@@ -111,7 +111,8 @@ def from_settings(cls, settings):
             producer = KafkaProducer(bootstrap_servers=settings['KAFKA_HOSTS'],
                                  retries=3,
                                  linger_ms=settings['KAFKA_PRODUCER_BATCH_LINGER_MS'],
-                                 buffer_memory=settings['KAFKA_PRODUCER_BUFFER_BYTES'])
+                                 buffer_memory=settings['KAFKA_PRODUCER_BUFFER_BYTES'],
+                                 value_serializer=lambda m: m.encode('utf-8'))
         except Exception as e:
                 logger.error("Unable to connect to Kafka in Pipeline"\
                     ", raising exit flag.")

crawler/crawling/redis_stats_middleware.py

@@ -41,7 +41,8 @@ def setup(self, settings):
         # set up redis
         self.redis_conn = redis.Redis(host=settings.get('REDIS_HOST'),
             port=settings.get('REDIS_PORT'),
-            db=settings.get('REDIS_DB'))
+            db=settings.get('REDIS_DB'),
+            decode_responses=True)
 
         try:
             self.redis_conn.info()

crawler/requirements.txt

@@ -28,7 +28,7 @@ requests==2.13.0 # Updated from 2.11.1
 requests-file==1.4.1 # Updated from 1.4
 retrying==1.3.3
 Scrapy==1.3.3
-scutils==1.2.0
+../utils # scutils==1.3.0dev0
 service-identity==16.0.0
 six==1.10.0
 testfixtures==4.13.5 # Updated from 4.10.0

crawler/tests/online.py

@@ -46,7 +46,8 @@ def setUp(self):
         # set up redis
         self.redis_conn = redis.Redis(host=self.settings['REDIS_HOST'],
                                       port=self.settings['REDIS_PORT'],
-                                      db=self.settings['REDIS_DB'])
+                                      db=self.settings['REDIS_DB'],
+                                      decode_responses=True)
         try:
             self.redis_conn.info()
         except ConnectionError:
@@ -66,7 +67,8 @@ def setUp(self):
             group_id="demo-id",
             auto_commit_interval_ms=10,
             consumer_timeout_ms=5000,
-            auto_offset_reset='earliest'
+            auto_offset_reset='earliest',
+            value_deserializer=lambda m: m.decode('utf-8')
         )
         time.sleep(1)
 

docker/crawler/Dockerfile

@@ -15,8 +15,10 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # install requirements
+COPY utils /usr/src/utils
 COPY crawler/requirements.txt /usr/src/app/
 RUN pip install --no-cache-dir -r requirements.txt
+RUN rm -rf /usr/src/utils
 
 # move codebase over
 COPY crawler /usr/src/app