Skip to content

HTTPS #1581

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Jan 3, 2016
Merged

HTTPS #1581

Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
c716574
Adding QiitaClient class
josenavas Dec 18, 2015
28f9697
Adding QiitaClient tests
josenavas Dec 18, 2015
15a6ee7
Using wss for websockets
josenavas Dec 18, 2015
7dbac08
Using https in the tornado webserver, adding default certificates and…
josenavas Dec 18, 2015
e7b541f
Update plugin to use https rather than http
josenavas Dec 18, 2015
c9a3da0
Type on config file
josenavas Dec 18, 2015
f60a98d
Removing prints
josenavas Dec 18, 2015
8739635
Fixing small bug
josenavas Dec 18, 2015
18fe70c
Adding documentation to the QiitaClient object
josenavas Dec 19, 2015
1337746
Addressing comments
josenavas Dec 19, 2015
d6f63dd
Addressing comments
josenavas Dec 21, 2015
0b848d8
Adding verbose comment to the _verify attribute in QiitaClient
josenavas Dec 21, 2015
e28153b
Removing try/except
josenavas Dec 21, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions qiita_core/configuration_manager.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,15 @@ def _get_main(self, config):
self.valid_upload_extension = []
print 'No files will be allowed to be uploaded.'

self.certificate_file = config.get('main', 'CERTIFICATE_FILE')
if not self.certificate_file:
self.certificate_file = join(install_dir, 'qiita_core',
'support_files', 'server.crt')
self.key_file = config.get('main', 'KEY_FILE')
if not self.key_file:
self.key_file = join(install_dir, 'qiita_core', 'support_files',
'server.key')

def _get_postgres(self, config):
"""Get the configuration of the postgres section"""
self.user = config.get('postgres', 'USER')
Expand Down
10 changes: 7 additions & 3 deletions qiita_core/support_files/config_test.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ LOG_PATH =
REQUIRE_APPROVAL = True

# Base URL: DO NOT ADD TRAILING SLASH
BASE_URL = http://localhost
BASE_URL = https://localhost

# Download path files
UPLOAD_DATA_DIR = /tmp/
Expand All @@ -39,12 +39,16 @@ BASE_DATA_DIR =
# Valid upload extension, comma separated. Empty for no uploads
VALID_UPLOAD_EXTENSION = fastq,fastq.gz,txt,tsv,sff,fna,qual

#Portal the site is working under
# Portal the site is working under
PORTAL = QIITA

#Script used for launching plugins
# Script used for launching plugins
PLUGIN_LAUNCHER = qiita-plugin-launcher

# Webserver certificate file paths
CERTIFICATE_FILE =
KEY_FILE =

# ----------------------------- SMTP settings -----------------------------
[smtp]
# The hostname to connect to
Expand Down
20 changes: 20 additions & 0 deletions qiita_core/support_files/server.crt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
-----BEGIN CERTIFICATE-----
MIIDVjCCAj4CCQCP4XnDqToF2zANBgkqhkiG9w0BAQUFADBtMQswCQYDVQQGEwJV
UzETMBEGA1UECBMKQ2FsaWZvcm5pYTESMBAGA1UEBxMJU2FuIERpZWdvMQ0wCwYD
VQQKEwRVQ1NEMRIwEAYDVQQLEwlLbmlnaHRMYWIxEjAQBgNVBAMTCWxvY2FsaG9z
dDAeFw0xNTEyMTgyMjE3MzBaFw0xNjEyMTcyMjE3MzBaMG0xCzAJBgNVBAYTAlVT
MRMwEQYDVQQIEwpDYWxpZm9ybmlhMRIwEAYDVQQHEwlTYW4gRGllZ28xDTALBgNV
BAoTBFVDU0QxEjAQBgNVBAsTCUtuaWdodExhYjESMBAGA1UEAxMJbG9jYWxob3N0
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAt1ggW4M/l3Wpru4+2Cro
nnqaUWD0ImLnkdAbmDjhGiCdKqdb8yzLeKipGaRY383gd5vMWHsKB1I3t+EzFWiY
fxd12Evx6MUIXVZSkdConk+8xlmJ5ba1Hgy7qzErY7+HOtgqm1ylyqTuOZyv3Umv
0W6ETLVz/alfzxTlqAkvuJn7I7RrbY81I3b5SOUxJTtj9pPwkZtVOD0ha3FH0LBu
lE4oi6rQQhzIbUDWLITZRCteplV5ikbC3JqaJ7pDiYnOIPnRR0UF+xdyTiOvSNH8
WrKuAdGGN+90PDt8fgQOwptE5l/RGyoJ2on7nlSj5crDtYzXXDYw0DCzuFG12nZV
FwIDAQABMA0GCSqGSIb3DQEBBQUAA4IBAQBTQJ8WYpSfsXsgmDa2uIYX5E+8ECGn
patQJuxYfOEp9knnBBe+QcaBMY6E7uH6EZz2QwS/gdhfY8e8QXw9sh9ZrQKQlIAK
Q5l5qxAtek0C90qdseYWoomBhpmqMUicF0OgecbdZ4X6Tfc4hvN5IXUTMn9ZJEaV
fduah3c7xEkSbHQl6iHnJswNKTc7Amm+BIwuYJjCZxVgKxAgvYzzg/TFU03gqzfE
h7ARs1p4WdHH+WTMqCZq8+sju3Lum4uwjYaiLaFE7psDkWWAYOu6Jv/o0V1zER/S
LzNaDfkm5kq4VURhPMQzdAiVdiTNKDFnLB3erg6wG95q5OiGNO1WYSw2
-----END CERTIFICATE-----
17 changes: 17 additions & 0 deletions qiita_core/support_files/server.csr
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
-----BEGIN CERTIFICATE REQUEST-----
MIICsjCCAZoCAQAwbTELMAkGA1UEBhMCVVMxEzARBgNVBAgTCkNhbGlmb3JuaWEx
EjAQBgNVBAcTCVNhbiBEaWVnbzENMAsGA1UEChMEVUNTRDESMBAGA1UECxMJS25p
Z2h0TGFiMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IB
DwAwggEKAoIBAQC3WCBbgz+Xdamu7j7YKuieeppRYPQiYueR0BuYOOEaIJ0qp1vz
LMt4qKkZpFjfzeB3m8xYewoHUje34TMVaJh/F3XYS/HoxQhdVlKR0KieT7zGWYnl
trUeDLurMStjv4c62CqbXKXKpO45nK/dSa/RboRMtXP9qV/PFOWoCS+4mfsjtGtt
jzUjdvlI5TElO2P2k/CRm1U4PSFrcUfQsG6UTiiLqtBCHMhtQNYshNlEK16mVXmK
RsLcmponukOJic4g+dFHRQX7F3JOI69I0fxasq4B0YY373Q8O3x+BA7Cm0TmX9Eb
KgnaifueVKPlysO1jNdcNjDQMLO4UbXadlUXAgMBAAGgADANBgkqhkiG9w0BAQUF
AAOCAQEAJD/s9Z150miwTYW08DklPoCjR1NijtFBUUkh176GCljNFe+rhnn2OLiV
skunJ4EiVAZ8O3bjgxGc53FnEMVrwisFugvnORTJAZpp7tKxBqpP++gqXf79wElN
ySTLjg9vFQgDjUxM2gO/8lH12HBw9xUDHFN2hOZ9ozPFf7KZUa6KUzh2AgHJ4UCF
9kFceNAvUamAHjv7BwDqoakLTuEBSZOycimDKiW8aBFGYCHFO2NMCzAKyVxmLtkB
K7+e5iWwvy3LewiPX/eK+jtRVwY+ujcvOZeCv6Th+oQv1BxstpJwOlca+5s1Rdke
kUu++nvrQUvWSsdQzxZq855r2TBA/w==
-----END CERTIFICATE REQUEST-----
27 changes: 27 additions & 0 deletions qiita_core/support_files/server.key
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
-----BEGIN RSA PRIVATE KEY-----
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have these been tested when serving from other systems? I want to make sure they don't trigger HTTPS error pages.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I try and access any page on the system, I get hte following error:

WARNING:tornado.general:SSL Error on 61 ('::1', 58512, 0, 0): [SSL: HTTP_REQUEST] http request (_ssl.c:590)

So these may not be good universally.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't test this in another system - that is what I exposed in the PR comments.

The error that you're seeing is because you are using http rather than https. Make sure to type https - in the live system the redirect is done through nginx, but in your local machine you need to type the https. Also make sure that your config file includes the https in the base_url, so when redirects happen in the system it works as expected.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done, and I now get the error I was expecting from chrome:

Your connection is not private

Attackers might be trying to steal your information from localhost (for example, passwords, messages, or credit cards). NET::ERR_CERT_AUTHORITY_INVALID

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you post a screenshot of the error rather than copy paste? I would like to see better the error from chrome.

Thanks,

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@biocore/qiita-dev can I have some comments here? The only alternative then is to ask each of our users to ask for a certificate to an authority - which can take some time and I don't think this is what we need to do. Without this, we can't use https neither oauth2.

I think that we good documentation this is not a problem - and we let the user know that is not a problem.

@squirrelo If you don't agree with this, please provide an alternative and constructive feedback.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should really default back to HTTP if we don't have a certificate provided, possibly with a warning when you start the webserver. This allows users to use standard HTTP when it's on their laptop and safe, and HTTPS in larger deploys.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This issue is a complicated one, on the one hand we want to maintain the installation process as simple as possible, but on the other hand we also want to have a consistent system that isn't raising flags and worrying our users. It almost feels like these two are incompatible, so these are the possible partial solutions I can think of:

  • Document this situation as part of the installation procedure, something along the lines of " ... you'll need to provide authorized and validated certificates, otherwise you may .... XXXXX".
  • Add a permanent warning to the system i.e. a bootstrap alert that never leaves the screen saying " your certificates are not xxxxx, to fix this xxxxx, or read a discussion on why this xxxxxxx".
  • Check the system before starting the webserver and halting execution if valid certificates are not found, this could be bypassed with a flag maybe something like --i-dont-care-about-security- or --local-system.
  • Add an installation manager that requires valid certificates to be present at installation time, so that the situation shown above cannot be reached.

The solution should probably be a combination of some of the points above.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just discussed some of this information with Jeff. Few points:

  • Valid certificates are not free: you have to buy them and their prices can be around ~$50. Even if it was $1 I don't think it is ok to ask our users running Qiita in their local machine to buy some certificate just because chrome complains about it.
  • The goal of a certificate is to let the user that the server is trusted - if we have documentation explaining that the server is your own machine, you can trust it as much as you trust your own machine.
  • There is no way of generating self-signed certificates that are recognized by the entire world unless you buy them. However, you can add an exception to your browser so it accepts such certificate.

About the comment of defaulting to HTTP: the entire work to use HTTPS has beed done so the RESTapi/plugins work as expected communicating through OAuth2 - if we default to HTTP, we can't use any of the plugins as they're authenticating through OAuth2, which requires HTTPS.

My vote goes top add detailed documentation in which we specify that if you are running locally in your machine, you will receive that warning the first time that you log in to the server (per @squirrelo 's observation, maybe every time that you start the server - I did not have that problem). Then, also add documentation that if you want to make your own Qiita server, you should get your own valid certificates from a Certificate Authority.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the solution is to improve documentation and/or try to do some of these suggestions: http://superuser.com/questions/772762/how-can-i-disable-security-checks-for-localhost

MIIEogIBAAKCAQEAt1ggW4M/l3Wpru4+2CronnqaUWD0ImLnkdAbmDjhGiCdKqdb
8yzLeKipGaRY383gd5vMWHsKB1I3t+EzFWiYfxd12Evx6MUIXVZSkdConk+8xlmJ
5ba1Hgy7qzErY7+HOtgqm1ylyqTuOZyv3Umv0W6ETLVz/alfzxTlqAkvuJn7I7Rr
bY81I3b5SOUxJTtj9pPwkZtVOD0ha3FH0LBulE4oi6rQQhzIbUDWLITZRCteplV5
ikbC3JqaJ7pDiYnOIPnRR0UF+xdyTiOvSNH8WrKuAdGGN+90PDt8fgQOwptE5l/R
GyoJ2on7nlSj5crDtYzXXDYw0DCzuFG12nZVFwIDAQABAoIBAAWPwsuD27X4d8E6
sOVVx+Cmz2+QhVdVsLiXm36umwVz39CfawahYTqzww1oEtoXIKwAKd61PMSkFdSa
HDaWCaoxR/3br+bjszFBoReTQMosmQlbRfeV50CWGktDHpHUCYIbdI9p7A2Nmzxc
KGmKBq9Pruu0xWP4c1OQyraiMLRkyjdPxV42t/cpolzkk+zt94om2DdmaZHgrwfM
TIC/PkgINlwIQfATFSaZSeTQ3bhfMzlLLaiDicf/ZUF8HnIToQjXwfApLv9/2DQB
BBjFXVXPODYADG0ddczX4GflO4cdyKeJALooQ2v1R7+F7XbTCHGSewFQsYN1cebP
Shnw4iECgYEA3t2wik92tLDD9yTw3Gl7KcoafW7yVMVjY7DrQqx5VLNN6H2ToIw0
1eaWx1xvabh8F4GyZ6hLqQJs2yOvuBcdEgvjs7rEAktT9w60MNQHP/9LxC7AloUR
QCpcp4kgf6QuYNBH5eeDMPXt/AyItlTBu1tLWvIyArZDFyFpQ1R+UFUCgYEA0po7
78yzQ2zTxxg+Zg1oFhojo8eqbBCxcnaeMMobaUvRlYdQn3FEgYEzAi39ajw+ApKO
megCBEc58mlWRaCEBzmOSdttCL/hZR5QD3tMyE97xZxu4qvty7mS8+Qx7VXi4E+R
9iB3QW2IdnO0ny0LL+/FRlseFd9Xi48HPEfGC7sCgYBZ4bC0Z03pm6y8/PIDAquA
dBjw45DqLV1jNFP9ZM/eIB5YmGS0iDc62S+QIbBxQ80QMeeAtLgm5mp+isrwhKnd
wFvJ/ZXVkwm/BvgXBTwTTGQDMBPN4MbPSQZrLvujXtSxiO6mQxN+6IBdebSdTFuB
s1UVzb3rqBQwvPHWDLqQrQKBgEQoemFfZFzb6aYzqG4qxJbymro0smkK9gLPdN7h
agGn8gpsoqEXbU5kW/fGno81M/RtLg2NUqnDy3MyuCMMke4sqGoqw5zjTLUHHG4G
P5FMeAg6WoCtl+YpBvA72BSxiR2DYv++/hmEs3dydvBVCZvYA4aDkrcNcy3Nek/+
ApBlAoGAQOXrFJa0kMKK89D6luYd7KWTHze7fSNmohxXKm4OW/9V/IxcDFASRasm
Ld45yQ6CRk42Ry0xA/TiEs9Gv8hPrJEzNXVzBMA+s0h6uY13p5cMgZdMA7XotM95
UgtsatiprRqzpYQItbPCYyPmDAscA+jsdoZzbXFjlOYz/9BZRCc=
-----END RSA PRIVATE KEY-----
3 changes: 2 additions & 1 deletion qiita_pet/templates/compute_wait.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
<script type="text/javascript">
$(document).ready(function() {

var host = 'ws://' + window.location.host + '/consumer/';
var socket_protocol = window.location.protocol == "https:" ? 'wss://' : 'ws://'
var host = socket_protocol + window.location.host + '/consumer/';
var websocket = new WebSocket(host);

// When the socket is opened, send a message from the client to the
Expand Down
23 changes: 9 additions & 14 deletions qiita_plugins/target_gene/tgp/pick_otus.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,7 @@
from functools import partial
from glob import glob

import requests

from tgp.util import (update_job_step, execute_request_retry, system_call,
format_payload)
from tgp.util import update_job_step, system_call, format_payload


def write_parameters_file(fp, parameters):
Expand Down Expand Up @@ -109,13 +106,13 @@ def generate_artifact_info(pick_out):
return [['BIOM', filepaths, True, True]]


def pick_closed_reference_otus(server_url, job_id, parameters, out_dir):
def pick_closed_reference_otus(qclient, job_id, parameters, out_dir):
"""Run split libraries fastq with the given parameters

Parameters
----------
server_url : str
The URL of the server
qclient : tgp.qiita_client.QiitaClient
The Qiita server client
job_id : str
The job id
parameters : dict
Expand All @@ -133,10 +130,9 @@ def pick_closed_reference_otus(server_url, job_id, parameters, out_dir):
ValueError
If there is any error gathering the information from the server
"""
update_job_step(server_url, job_id, "Step 1 of 3: Collecting information")
update_job_step(qclient, job_id, "Step 1 of 3: Collecting information")
artifact_id = parameters['input_data']
url = "%s/qiita_db/artifacts/%s/filepaths/" % (server_url, artifact_id)
fps_info = execute_request_retry(requests.get, url)
fps_info = qclient.get("/qiita_db/artifacts/%s/filepaths/" % artifact_id)
if not fps_info or not fps_info['success']:
error_msg = "Could not get artifact filepath information: %s"
if fps_info:
Expand All @@ -147,8 +143,7 @@ def pick_closed_reference_otus(server_url, job_id, parameters, out_dir):
fps = fps_info['filepaths']

reference_id = parameters['reference']
url = "%s/qiita_db/references/%s/filepaths/" % (server_url, reference_id)
ref_info = execute_request_retry(requests.get, url)
ref_info = qclient.get("/qiita_db/references/%s/filepaths/" % reference_id)
if not ref_info or not ref_info['success']:
error_msg = "Could not get artifact filepath information: %s"
if ref_info:
Expand All @@ -158,11 +153,11 @@ def pick_closed_reference_otus(server_url, job_id, parameters, out_dir):
raise ValueError(error_msg)
reference_fps = ref_info['filepaths']

update_job_step(server_url, job_id, "Step 2 of 3: Generating command")
update_job_step(qclient, job_id, "Step 2 of 3: Generating command")
command, pick_out = generate_pick_closed_reference_otus_cmd(
fps, out_dir, parameters, reference_fps)

update_job_step(server_url, job_id, "Step 3 of 3: Executing OTU picking")
update_job_step(qclient, job_id, "Step 3 of 3: Executing OTU picking")
std_out, std_err, return_value = system_call(command)
if return_value != 0:
error_msg = ("Error running OTU picking:\nStd out: %s\nStd err: %s"
Expand Down
18 changes: 8 additions & 10 deletions qiita_plugins/target_gene/tgp/plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,10 +11,8 @@
from os.path import exists
from os import makedirs

import requests

from tgp.util import (start_heartbeat, complete_job, execute_request_retry,
format_payload)
from tgp.qiita_client import QiitaClient
from tgp.util import start_heartbeat, complete_job, format_payload
from tgp.split_libraries import split_libraries, split_libraries_fastq
from tgp.pick_otus import pick_closed_reference_otus

Expand All @@ -40,27 +38,27 @@ def execute_job(server_url, job_id, output_dir):
RuntimeError
If there is a problem gathering the job information
"""
qclient = QiitaClient(server_url)
# Request job information
url = "%s/qiita_db/jobs/%s" % (server_url, job_id)
job_info = execute_request_retry(requests.get, url)
job_info = qclient.get("/qiita_db/jobs/%s" % job_id)
# Check if we have received the job information so we can start it
if job_info['success']:
if job_info and job_info['success']:
# Starting the heartbeat
start_heartbeat(server_url, job_id)
start_heartbeat(qclient, job_id)
# Execute the given task
task_name = job_info['command']
task = TASK_DICT[task_name]

if not exists(output_dir):
makedirs(output_dir)
try:
payload = task(server_url, job_id, job_info['parameters'],
payload = task(qclient, job_id, job_info['parameters'],
output_dir)
except Exception:
exc_str = repr(traceback.format_exception(*sys.exc_info()))
error_msg = ("Error executing %s:\n%s" % (task_name, exc_str))
payload = format_payload(False, error_msg=error_msg)
# The job completed
complete_job(server_url, job_id, payload)
complete_job(qclient, job_id, payload)
else:
raise RuntimeError("Can't get job (%s) information" % job_id)
Loading