From d7fc1f4a59f438d7c79be338da6d13faff590901 Mon Sep 17 00:00:00 2001 From: Teemu Erkkola Date: Wed, 4 Oct 2017 13:54:41 +0300 Subject: [PATCH] Added datapusher and related configuration --- ansible/cluster-webserver.yml | 1 + ansible/roles/datapusher/defaults/main.yml | 1 + ansible/roles/datapusher/tasks/main.yml | 27 ++++++++++++++++ .../roles/datapusher/templates/datapusher.j2 | 23 +++++++++++++ .../datapusher/templates/datapusher.wsgi.j2 | 14 ++++++++ .../templates/datapusher_settings.py.j2 | 32 +++++++++++++++++++ ansible/single-server.yml | 1 + ansible/vars/common.yml | 3 +- 8 files changed, 101 insertions(+), 1 deletion(-) create mode 100644 ansible/roles/datapusher/defaults/main.yml create mode 100644 ansible/roles/datapusher/tasks/main.yml create mode 100644 ansible/roles/datapusher/templates/datapusher.j2 create mode 100644 ansible/roles/datapusher/templates/datapusher.wsgi.j2 create mode 100644 ansible/roles/datapusher/templates/datapusher_settings.py.j2 diff --git a/ansible/cluster-webserver.yml b/ansible/cluster-webserver.yml index 7a10edbbc2..0b5a8dcf60 100644 --- a/ansible/cluster-webserver.yml +++ b/ansible/cluster-webserver.yml @@ -24,6 +24,7 @@ - ckan-restart - drupal - data + - datapusher handlers: - name: restart sshd service: name=ssh state=restarted diff --git a/ansible/roles/datapusher/defaults/main.yml b/ansible/roles/datapusher/defaults/main.yml new file mode 100644 index 0000000000..41619cec6e --- /dev/null +++ b/ansible/roles/datapusher/defaults/main.yml @@ -0,0 +1 @@ +datapusher_max_content_length: "{{ 500 * 1024 * 1024 }}" #500MB \ No newline at end of file diff --git a/ansible/roles/datapusher/tasks/main.yml b/ansible/roles/datapusher/tasks/main.yml new file mode 100644 index 0000000000..bff4912895 --- /dev/null +++ b/ansible/roles/datapusher/tasks/main.yml @@ -0,0 +1,27 @@ +- name: Download Datapusher + git: + repo: https://github.com/6aika/datapusher.git + dest: "{{ cache_path }}/datapusher.git" + bare: yes + +- name: Install Datapusher + pip: name="git+file://{{ cache_path }}/datapusher.git@master#egg=datapusher" virtualenv={{ datapusherenv }} extra_args="--exists-action=s -e" + +- name: Install Datapusher requirements + pip: requirements={{ datapusherenv }}/src/datapusher/requirements.txt virtualenv={{ datapusherenv }} state=latest + +- name: Link datapusher sources + command: "{{ datapusherenv }}/bin/python setup.py develop chdir={{ datapusherenv }}/src/datapusher" + +- name: Copy Datapusher site file + template: src={{ item.src }} dest={{ item.dest }} mode={{ item.mode }} owner={{ item.owner }} group={{ item.group }} + with_items: + - { src: datapusher.j2, dest: /etc/apache2/sites-available/datapusher.conf, mode: "0644", owner: root, group: root} + - { src: datapusher.wsgi.j2, dest: /etc/ckan/default/datapusher.wsgi, mode: "0640", owner: root, group: "{{ www_group }}" } + - { src: datapusher_settings.py.j2, dest: /etc/ckan/default/datapusher_settings.py, mode: "0640", owner: root, group: "{{ www_group }}" } + +- name: Enable Datapusher configuration + file: src=/etc/apache2/sites-available/datapusher.conf dest=/etc/apache2/sites-enabled/datapusher.conf state=link owner=root group=root + +- name: Restart Apache + service: name=apache2 state=restarted diff --git a/ansible/roles/datapusher/templates/datapusher.j2 b/ansible/roles/datapusher/templates/datapusher.j2 new file mode 100644 index 0000000000..cdfb68ffb8 --- /dev/null +++ b/ansible/roles/datapusher/templates/datapusher.j2 @@ -0,0 +1,23 @@ + + + ServerName ckan + + # this is our app + WSGIScriptAlias / /etc/ckan/default/datapusher.wsgi + + # pass authorization info on (needed for rest api) + WSGIPassAuthorization On + + # Deploy as a daemon (avoids conflicts between CKAN instances) + WSGIDaemonProcess datapusher display-name=demo processes=1 threads=15 + + WSGIProcessGroup datapusher + + ErrorLog /var/log/apache2/datapusher.error.log + CustomLog /var/log/apache2/datapusher.custom.log combined + + + Require all granted + + + \ No newline at end of file diff --git a/ansible/roles/datapusher/templates/datapusher.wsgi.j2 b/ansible/roles/datapusher/templates/datapusher.wsgi.j2 new file mode 100644 index 0000000000..a77d9fa343 --- /dev/null +++ b/ansible/roles/datapusher/templates/datapusher.wsgi.j2 @@ -0,0 +1,14 @@ +import os +import sys +import hashlib + +activate_this = os.path.join('/usr/lib/ckan/datapusher/bin/activate_this.py') +execfile(activate_this, dict(__file__=activate_this)) + +import ckanserviceprovider.web as web +os.environ['JOB_CONFIG'] = '/etc/ckan/default/datapusher_settings.py' +web.init() + +import datapusher.jobs as jobs + +application = web.app \ No newline at end of file diff --git a/ansible/roles/datapusher/templates/datapusher_settings.py.j2 b/ansible/roles/datapusher/templates/datapusher_settings.py.j2 new file mode 100644 index 0000000000..88df91f8b6 --- /dev/null +++ b/ansible/roles/datapusher/templates/datapusher_settings.py.j2 @@ -0,0 +1,32 @@ +import uuid + +DEBUG = False +TESTING = False +SECRET_KEY = str(uuid.uuid4()) +USERNAME = str(uuid.uuid4()) +PASSWORD = str(uuid.uuid4()) + +NAME = 'datapusher' + +# database + +SQLALCHEMY_DATABASE_URI = 'sqlite:////tmp/job_store.db' + +# webserver host and port + +HOST = '0.0.0.0' +PORT = 8800 + +# logging + +#FROM_EMAIL = 'server-error@example.com' +#ADMINS = ['yourname@example.com'] # where to send emails + +#LOG_FILE = '/tmp/ckan_service.log' +STDERR = True + +{% if deployment_environment_id == "vagrant" %} +SSL_VERIFY = 'False' +{% endif %} + +MAX_CONTENT_LENGTH = {{ datapusher_max_content_length }} diff --git a/ansible/single-server.yml b/ansible/single-server.yml index 7b744626e6..6286b9c97e 100644 --- a/ansible/single-server.yml +++ b/ansible/single-server.yml @@ -35,6 +35,7 @@ - drupal - clamav - data + - datapusher handlers: - name: restart sshd service: name=ssh state=restarted diff --git a/ansible/vars/common.yml b/ansible/vars/common.yml index 6790e5cb1b..2466c7bc3f 100644 --- a/ansible/vars/common.yml +++ b/ansible/vars/common.yml @@ -72,6 +72,7 @@ ckan_admins: ckan_ini: /etc/ckan/default/production.ini virtual_environment: /usr/lib/ckan/default +datapusherenv: /usr/lib/ckan/datapusher ckan_who_ini: /etc/ckan/default/who.ini harvest_sources: false @@ -161,4 +162,4 @@ AWS: use_iam_role: false access_key: somekey secret_access_key: somesecret - ckan_s3_bucket: somebucket \ No newline at end of file + ckan_s3_bucket: somebucket