removed beanstalkc

.travis.yml

@@ -5,8 +5,8 @@ python:
   - 3.4
   - 3.5
   - 3.6
-  #- 3.7
-  #- 3.8
+  - 3.7
+  - 3.8
 services:
     - docker
     - mongodb
@@ -27,9 +27,6 @@ before_install:
     - sudo apt-get update -qq
     - sudo apt-get install -y couchdb
     - sudo systemctl start couchdb
-    - sudo apt-get install -y beanstalkd libgnutls28-dev
-    - echo "START=yes" | sudo tee -a /etc/default/beanstalkd > /dev/null
-    - sudo service beanstalkd start
     - curl -O https://download.elastic.co/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.4.0/elasticsearch-2.4.0.deb && sudo dpkg -i --force-confnew elasticsearch-2.4.0.deb && sudo service elasticsearch restart
     - npm install express puppeteer
     - sudo docker pull scrapinghub/splash

README.md

@@ -6,7 +6,7 @@ A Powerful Spider(Web Crawler) System in Python. **[TRY IT NOW!][Demo]**
 - Write script in Python
 - Powerful WebUI with script editor, task monitor, project manager and result viewer
 - [MySQL](https://www.mysql.com/), [MongoDB](https://www.mongodb.org/), [Redis](http://redis.io/), [SQLite](https://www.sqlite.org/), [Elasticsearch](https://www.elastic.co/products/elasticsearch); [PostgreSQL](http://www.postgresql.org/) with [SQLAlchemy](http://www.sqlalchemy.org/) as database backend
-- [RabbitMQ](http://www.rabbitmq.com/), [Beanstalk](http://kr.github.com/beanstalkd/), [Redis](http://redis.io/) and [Kombu](http://kombu.readthedocs.org/) as message queue
+- [RabbitMQ](http://www.rabbitmq.com/), [Redis](http://redis.io/) and [Kombu](http://kombu.readthedocs.org/) as message queue
 - Task priority, retry, periodical, recrawl by age, etc...
 - Distributed architecture, Crawl Javascript pages, Python 2.{6,7}, 3.{3,4,5,6} support, etc...
 

docs/Command-Line.md

@@ -90,8 +90,6 @@ type:
 rabbitmq:
     amqp://username:password@host:5672/%2F
     see https://www.rabbitmq.com/uri-spec.html
-beanstalk:
-    beanstalk://host:11300/
 redis:
     redis://host:6379/db
     redis://host1:port1,host2:port2,...,hostn:portn (for redis 3.x in cluster mode)

docs/Deployment.md

@@ -10,7 +10,7 @@ Installation
 
 To deploy pyspider components in each single processes, you need at least one database service. pyspider now supports [MySQL](http://www.mysql.com/), [MongoDB](http://www.mongodb.org/) and [PostgreSQL](http://www.postgresql.org/). You can choose one of them.
 
-And you need a message queue service to connect the components together. You can use [RabbitMQ](http://www.rabbitmq.com/), [Beanstalk](http://kr.github.io/beanstalkd/) or [Redis](http://redis.io/) as message queue.
+And you need a message queue service to connect the components together. You can use [RabbitMQ](http://www.rabbitmq.com/) or [Redis](http://redis.io/) as message queue.
 
 `pip install --allow-all-external pyspider[all]`
 
@@ -81,8 +81,6 @@ You can use connection URL to specify the message queue:
 rabbitmq:
     amqp://username:password@host:5672/%2F
     Refer: https://www.rabbitmq.com/uri-spec.html
-beanstalk:
-    beanstalk://host:11300/
 redis:
     redis://host:6379/db
     redis://host1:port1,host2:port2,...,hostn:portn (for redis 3.x in cluster mode)

docs/index.md

@@ -6,7 +6,7 @@ A Powerful Spider(Web Crawler) System in Python. **[TRY IT NOW!][Demo]**
 - Write script in Python
 - Powerful WebUI with script editor, task monitor, project manager and result viewer
 - [MySQL](https://www.mysql.com/), [MongoDB](https://www.mongodb.org/), [Redis](http://redis.io/), [SQLite](https://www.sqlite.org/), [Elasticsearch](https://www.elastic.co/products/elasticsearch); [PostgreSQL](http://www.postgresql.org/) with [SQLAlchemy](http://www.sqlalchemy.org/) as database backend
-- [RabbitMQ](http://www.rabbitmq.com/), [Beanstalk](http://kr.github.com/beanstalkd/), [Redis](http://redis.io/) and [Kombu](http://kombu.readthedocs.org/) as message queue
+- [RabbitMQ](http://www.rabbitmq.com/), [Redis](http://redis.io/) and [Kombu](http://kombu.readthedocs.org/) as message queue
 - Task priority, retry, periodical, recrawl by age, etc...
 - Distributed architecture, Crawl Javascript pages, Python 2&3, etc...
 

pyspider/message_queue/__init__.py

@@ -23,8 +23,6 @@ def connect_message_queue(name, url=None, maxsize=0, lazy_limit=True):
     rabbitmq:
         amqp://username:password@host:5672/%2F
         see https://www.rabbitmq.com/uri-spec.html
-    beanstalk:
-        beanstalk://host:11300/
     redis:
         redis://host:6379/db
         redis://host1:port1,host2:port2,...,hostn:portn (for redis 3.x in cluster mode)
@@ -43,9 +41,6 @@ def connect_message_queue(name, url=None, maxsize=0, lazy_limit=True):
     if parsed.scheme == 'amqp':
         from .rabbitmq import Queue
         return Queue(name, url, maxsize=maxsize, lazy_limit=lazy_limit)
-    elif parsed.scheme == 'beanstalk':
-        from .beanstalk import Queue
-        return Queue(name, host=parsed.netloc, maxsize=maxsize)
     elif parsed.scheme == 'redis':
         from .redis_queue import Queue
         if ',' in parsed.netloc:

pyspider/run.py

@@ -145,8 +145,6 @@ def cli(ctx, **kwargs):
     elif os.environ.get('RABBITMQ_NAME'):
         kwargs['message_queue'] = ("amqp://guest:guest@%(RABBITMQ_PORT_5672_TCP_ADDR)s"
                                    ":%(RABBITMQ_PORT_5672_TCP_PORT)s/%%2F" % os.environ)
-    elif kwargs.get('beanstalk'):
-        kwargs['message_queue'] = "beanstalk://%s/" % kwargs['beanstalk']
 
     for name in ('newtask_queue', 'status_queue', 'scheduler2fetcher',
                  'fetcher2processor', 'processor2result'):

setup.py

@@ -32,24 +32,12 @@
     'tblib==1.4.0'
 ]
 
-if sys.version_info < (2, 7):  # 2.6
-    install_requires.extend([
-        'wsgidav<2.0.0',
-        'tornado>=3.2,<4.5',
-        'pyquery<1.3.0',
-    ])
-elif sys.version_info >= (3, 0):  # 3.*
+if sys.version_info >= (3, 0):  # 3.*
     install_requires.extend([
         'wsgidav==2.3.0',
         'tornado>=3.2,<=4.5.3',
         'pyquery',
     ])
-else:  # 2.7
-    install_requires.extend([
-        'wsgidav',
-        'tornado>=3.2,<=4.5.3',
-        'pyquery',
-    ])
 
 extras_require_all = [
     'mysql-connector-python==8.0.16',
@@ -59,31 +47,13 @@
     'psycopg2==2.8.2',
     'elasticsearch==2.3.0',
 ]
-if sys.version_info < (2, 7):  # 2.6
-    extras_require_all.extend([
-        'kombu<4.0',
-        'amqp>=1.3.0,<2.0',
-        'pika>=0.9.14',
-        'beanstalkc',
-        'SQLAlchemy>=0.9.7,<=1.1.13',
-        'unittest2>=0.5.1',
-    ])
-elif sys.version_info >= (3, 0):  # 3.*
+if sys.version_info >= (3, 0):  # 3.*
     extras_require_all.extend([
         'kombu==4.4.0',
         'amqp==2.4.0',
         'SQLAlchemy==1.3.10',
         'pika==1.1.0'
     ])
-else:  # 2.7
-    extras_require_all.extend([
-        'kombu',
-        'pika>=0.9.14',
-        'beanstalkc',
-        'amqp>=1.3.0',
-        'SQLAlchemy>=0.9.7',
-        'unittest2>=0.5.1',
-    ])
 
 
 setup(
@@ -102,13 +72,13 @@
 
     classifiers=[
         'Development Status :: 4 - Beta',
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.6',
-        'Programming Language :: Python :: 2.7',
         'Programming Language :: Python :: 3',
         'Programming Language :: Python :: 3.3',
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
 
         'License :: OSI Approved :: Apache Software License',
 

tests/test_message_queue.py

@@ -159,36 +159,6 @@ def test_30_full(self):
         with self.assertRaises(Queue.Full):
             self.q1.put_nowait('TEST_DATA6')
 
-#@unittest.skipIf(True, "beanstalk queue can't pass the test currently")
-@unittest.skipIf(six.PY3, 'beanstalkc not suport python 3')
-@unittest.skipIf(os.environ.get('IGNORE_BEANSTALK') or os.environ.get('IGNORE_ALL'), 'no beanstalk server for test.')
-class TestBeansTalkQueue(TestMessageQueue, unittest.TestCase):
-
-    @classmethod
-    def setUpClass(self):
-        from pyspider.message_queue import connect_message_queue
-        with utils.timeout(3):
-            self.q1 = connect_message_queue('test_queue', 'beanstalk://localhost:11300',
-                                            maxsize=5)
-            self.q2 = connect_message_queue('test_queue', 'beanstalk://localhost:11300',
-                                            maxsize=5)
-            self.q3 = connect_message_queue('test_queue_for_threading_test',
-                                            'beanstalk://localhost:11300')
-            while not self.q1.empty():
-                self.q1.get()
-            while not self.q2.empty():
-                self.q2.get()
-            while not self.q3.empty():
-                self.q3.get()
-
-    @classmethod
-    def tearDownClass(self):
-        while not self.q1.empty():
-            self.q1.get()
-        while not self.q2.empty():
-            self.q2.get()
-        while not self.q3.empty():
-            self.q3.get()
 
 @unittest.skipIf(os.environ.get('IGNORE_REDIS') or os.environ.get('IGNORE_ALL'), 'no redis server for test.')
 class TestRedisQueue(TestMessageQueue, unittest.TestCase):
@@ -257,11 +227,6 @@ class TestKombuAmpqQueue(TestKombuQueue):
 class TestKombuRedisQueue(TestKombuQueue):
     kombu_url = 'kombu+redis://'
 
-@unittest.skip('test cannot pass, get is buffered')
-@unittest.skipIf(os.environ.get('IGNORE_BEANSTALK') or os.environ.get('IGNORE_ALL'), 'no beanstalk server for test.')
-class TestKombuBeanstalkQueue(TestKombuQueue):
-    kombu_url = 'kombu+beanstalk://'
-
 @unittest.skipIf(os.environ.get('IGNORE_MONGODB') or os.environ.get('IGNORE_ALL'), 'no mongodb server for test.')
 class TestKombuMongoDBQueue(TestKombuQueue):
     kombu_url = 'kombu+mongodb://'