Added datapusher and related configuration

ansible/cluster-webserver.yml

@@ -24,6 +24,7 @@
     - ckan-restart
     - drupal
     - data
+    - datapusher
   handlers:
     - name: restart sshd
       service: name=ssh state=restarted

ansible/roles/datapusher/defaults/main.yml

@@ -0,0 +1 @@
+datapusher_max_content_length: "{{ 500 * 1024 * 1024 }}" #500MB

ansible/roles/datapusher/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: Download Datapusher
+  git:
+    repo: https://github.com/6aika/datapusher.git
+    dest: "{{ cache_path }}/datapusher.git"
+    bare: yes
+
+- name: Install Datapusher
+  pip: name="git+file://{{ cache_path }}/datapusher.git@master#egg=datapusher" virtualenv={{ datapusherenv }} extra_args="--exists-action=s -e"
+
+- name: Install Datapusher requirements
+  pip: requirements={{ datapusherenv }}/src/datapusher/requirements.txt virtualenv={{ datapusherenv }} state=latest
+
+- name: Link datapusher sources
+  command: "{{ datapusherenv }}/bin/python setup.py develop chdir={{ datapusherenv }}/src/datapusher"
+
+- name: Copy Datapusher site file
+  template: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} owner={{ item.owner }} group={{ item.group }}
+  with_items:
+      - { src: datapusher.j2, dest: /etc/apache2/sites-available/datapusher.conf, mode: "0644", owner: root, group: root}
+      - { src: datapusher.wsgi.j2, dest: /etc/ckan/default/datapusher.wsgi, mode: "0640", owner: root, group: "{{ www_group }}" }
+      - { src: datapusher_settings.py.j2, dest: /etc/ckan/default/datapusher_settings.py, mode: "0640", owner: root, group: "{{ www_group }}" }
+
+- name: Enable Datapusher configuration
+  file: src=/etc/apache2/sites-available/datapusher.conf dest=/etc/apache2/sites-enabled/datapusher.conf state=link owner=root group=root
+
+- name: Restart Apache
+  service: name=apache2 state=restarted

ansible/roles/datapusher/templates/datapusher.j2

@@ -0,0 +1,23 @@
+<VirtualHost *:8800>
+
+    ServerName ckan
+
+    # this is our app
+    WSGIScriptAlias / /etc/ckan/default/datapusher.wsgi
+
+    # pass authorization info on (needed for rest api)
+    WSGIPassAuthorization On
+
+    # Deploy as a daemon (avoids conflicts between CKAN instances)
+    WSGIDaemonProcess datapusher display-name=demo processes=1 threads=15
+
+    WSGIProcessGroup datapusher
+
+    ErrorLog /var/log/apache2/datapusher.error.log
+    CustomLog /var/log/apache2/datapusher.custom.log combined
+
+    <Directory "/" >
+        Require all granted
+    </Directory>
+
+</VirtualHost>

ansible/roles/datapusher/templates/datapusher.wsgi.j2

@@ -0,0 +1,14 @@
+import os
+import sys
+import hashlib
+
+activate_this = os.path.join('/usr/lib/ckan/datapusher/bin/activate_this.py')
+execfile(activate_this, dict(__file__=activate_this))
+
+import ckanserviceprovider.web as web
+os.environ['JOB_CONFIG'] = '/etc/ckan/default/datapusher_settings.py'
+web.init()
+
+import datapusher.jobs as jobs
+
+application = web.app

ansible/roles/datapusher/templates/datapusher_settings.py.j2

@@ -0,0 +1,32 @@
+import uuid
+
+DEBUG = False
+TESTING = False
+SECRET_KEY = str(uuid.uuid4())
+USERNAME = str(uuid.uuid4())
+PASSWORD = str(uuid.uuid4())
+
+NAME = 'datapusher'
+
+# database
+
+SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/job_store.db'
+
+# webserver host and port
+
+HOST = '0.0.0.0'
+PORT = 8800
+
+# logging
+
+#FROM_EMAIL = 'server-error@example.com'
+#ADMINS = ['yourname@example.com']  # where to send emails
+
+#LOG_FILE = '/tmp/ckan_service.log'
+STDERR = True
+
+{% if deployment_environment_id == "vagrant" %}
+SSL_VERIFY = 'False'
+{% endif %}
+
+MAX_CONTENT_LENGTH = {{ datapusher_max_content_length }}

ansible/single-server.yml

@@ -35,6 +35,7 @@
     - drupal
     - clamav
     - data
+    - datapusher
   handlers:
     - name: restart sshd
       service: name=ssh state=restarted

ansible/vars/common.yml

@@ -72,6 +72,7 @@ ckan_admins:
 
 ckan_ini: /etc/ckan/default/production.ini
 virtual_environment: /usr/lib/ckan/default
+datapusherenv: /usr/lib/ckan/datapusher
 ckan_who_ini: /etc/ckan/default/who.ini
 
 harvest_sources: false
@@ -161,4 +162,4 @@ AWS:
   use_iam_role: false
   access_key: somekey
   secret_access_key: somesecret
-  ckan_s3_bucket: somebucket
+  ckan_s3_bucket: somebucket