Remove celery from requirements, its dependencies and all references

Celery is no longer necessary now that the daemon is daemonized
by the circus library

.travis-data/test_daemon.py

@@ -12,6 +12,7 @@
 import sys
 import time
 from aiida.common.exceptions import NotExistent
+from aiida.daemon.client import ProfileDaemonClient
 from aiida.orm import DataFactory
 from aiida.orm.data.int import Int
 from aiida.work.launch import run_get_node, submit
@@ -25,12 +26,13 @@
 number_workchains = 8 # Number of workchains to submit
 
 def print_daemon_log():
-    home = os.environ['HOME']
-    print "Output of 'cat {}/.aiida/daemon/log/celery.log':".format(home)
+    profile_daemon_client = ProfileDaemonClient()
+    daemon_log = profile_daemon_client.daemon_log_file
+
+    print "Output of 'cat {}':".format(daemon_log)
     try:
         print subprocess.check_output(
-            ["cat", "{}/.aiida/daemon/log/celery.log".format(home)],
-            stderr=subprocess.STDOUT,
+            ['cat', '{}'.format(daemon_log)], stderr=subprocess.STDOUT,
         )
     except subprocess.CalledProcessError as e:
         print "Note: the command failed, message: {}".format(e.message)

aiida/backends/djsite/settings/settings.py

@@ -98,8 +98,7 @@
 
 # Usual Django settings starts here.............
 
-# Keep it to False! Otherwise every query is stored
-# in memory and looks like a memory leak with celery
+# Keep it to False! Otherwise every query is stored in memory
 DEBUG = False
 TEMPLATE_DEBUG = DEBUG
 

aiida/common/log.py

@@ -143,9 +143,9 @@ def configure_logging(daemon=False, daemon_log_file=None):
     """
     Setup the logging by retrieving the LOGGING dictionary from aiida and passing it to
     the python module logging.config.dictConfig. If the logging needs to be setup for the
-    daemon running a task for one of the celery workers, set the argument 'daemon' to True
-    and specify the path to the log file. This will cause a 'daemon_handler' to be added
-    to all the configured loggers, that is a RotatingFileHandler that writes to the log file.
+    daemon, set the argument 'daemon' to True and specify the path to the log file. This
+    will cause a 'daemon_handler' to be added to all the configured loggers, that is a
+    RotatingFileHandler that writes to the log file.
 
     :param daemon: configure the logging for a daemon task by adding a file handler instead
         of the default 'console' StreamHandler

aiida/common/setup.py

@@ -852,12 +852,12 @@ class _NoDefaultValue(object):
         "Minimum level to log to the console",
         "WARNING",
         ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"]),
-    "logging.celery_loglevel": (
-        "logging_celery_log_level",
+    "logging.circus_loglevel": (
+        "logging_circus_log_level",
         "string",
-        "Minimum level to log to the file ~/.aiida/daemon/log/aiida_daemon.log "
-        "for the 'celery' logger",
-        "WARNING",
+        "Minimum level to log to the circus daemon log file"
+        "for the 'circus' logger",
+        "INFO",
         ["CRITICAL", "ERROR", "WARNING", "REPORT", "INFO", "DEBUG"]),
     "logging.db_loglevel": (
         "logging_db_log_level",

docs/requirements_for_rtd.txt

@@ -13,10 +13,7 @@ SQLAlchemy==1.0.12
 Sphinx==1.5.2
 aldjemy==0.6.0
 alembic==0.9.6
-amqp==1.4.9
 anyjson==0.3.3
-billiard==3.3.0.23
-celery==3.1.25
 circus>=0.14.0
 click-plugins
 click-spinner

docs/source/developer_guide/database_schema.rst

@@ -16,11 +16,6 @@ highlighted with light green and the columns that participate to an index
 
 We will now briefly describe some of the database tables.
 
-.. note:: the ``kombu_queue`` and the ``kombu_message`` tables are generated by
-          the celery and not by the AiiDA. In AiiDA the celery is used by daemon to
-          handle the distributed task queue and real-time messaging. To know
-          more about celery please refer `celery documentation <http://docs.celeryproject.org/en/latest/>`_.
-
 db_dbnode
 ---------
 This table contains the information about all the Node instances stored in

docs/source/install/installation.rst

@@ -220,8 +220,6 @@ For Mac OS X systems, the path to add is usually ``~/Library/Python/2.7/bin``::
 
     export PATH=~/Library/Python/2.7/bin:${PATH}
 
-To verify if this is the correct path to add, navigate to this location and you should find the executable ``supervisord``, or ``celeryd``, in the directory.
-
 After updating your ``PATH`` you can check if it worked in the following way:
 
 * type ``verdi`` on your terminal, and check if the program starts (it should

docs/source/troubleshooting/index.rst

@@ -15,18 +15,20 @@ Increasing the debug level
 ==========================
 
 By default, the logging level of AiiDA is minimal to avoid filling logfiles.
-Only warnings and errors are logged (to the
-``~/.aiida/daemon/log/aiida_daemon.log`` file), while info and debug
+Only warnings and errors are logged to the daemon log files, while info and debug
 messages are discarded.
 
 If you are experiencing a problem, you can change the default minimum logging
-level of AiiDA messages (and celery messages -- celery is the library that we
+level of AiiDA messages (and circus messages -- circus is the library that we
 use to manage the daemon process) using, on the command line, the two
 following commands::
 
-  verdi devel setproperty logging.celery_loglevel DEBUG
+  verdi devel setproperty logging.circus_loglevel DEBUG
   verdi devel setproperty logging.aiida_loglevel DEBUG
 
+For each profile that runs a daemon, there will be two unique logfiles, one for
+AiiDA log messages and one from the circus daemonizer. These files can be found
+in the ``.aiida`` log folder, which by default can be found at ``~/.aiida/daemon/log/aiida_daemon.log``.
 After rebooting the daemon (``verdi daemon restart``), the number of messages
 logged will increase significantly and may help in understanding
 the source of the problem. 
@@ -38,7 +40,7 @@ the source of the problem.
 .. note:: When the problem is solved, we suggest to bring back the default
   logging level, using the two commands::
 
-    verdi devel delproperty logging.celery_loglevel
+    verdi devel delproperty logging.circus_loglevel
     verdi devel delproperty logging.aiida_loglevel
 
   to avoid to fill the logfiles.

requirements.txt

@@ -1,7 +1,6 @@
 alabaster==0.7.10
 aldjemy==0.6.0
 alembic==0.9.6
-amqp==1.4.9
 aniso8601==2.0.0
 anyjson==0.3.3
 ase==3.12.0
@@ -12,9 +11,7 @@ Babel==2.5.3
 backports.functools-lru-cache==1.4
 backports.shutil-get-terminal-size==1.0.0
 bcrypt==3.1.4
-billiard==3.3.0.23
 cached-property==1.3.1
-celery==3.1.25
 certifi==2018.1.18
 cffi==1.11.4
 chardet==3.0.4

setup_requirements.py

@@ -23,20 +23,6 @@
     'six==1.11.0',
     'future==0.16.0',
     'pathlib2==2.3.0',
-    # We need for the time being to stay with an old version
-    # of celery, including the versions of the AMQP libraries below,
-    # because the support for a SQLA broker has been dropped in later
-    # versions... Actually, this might be source or problems with
-    # SQLA for us... probably switch to using rabbitmq?
-    # Note that however this requires a new server process.
-    # We also need to fix the celery version because support for Django 1.7
-    # was dropped from celery 3.2
-    'celery==3.1.25',
-    # The next two are internal dependencies of celery, but since
-    # in the past we had version mismatch problems, we freeze them
-    # as well
-    'billiard==3.3.0.23',
-    'amqp==1.4.9',
     'anyjson==0.3.3',
     'psutil==5.4.0',
     'meld3==1.0.0',