Skip to content

Join using instead on db issues #1126

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ Qiita
[![Build Status](https://travis-ci.org/biocore/qiita.png?branch=master)](https://travis-ci.org/biocore/qiita)
[![Coverage Status](https://coveralls.io/repos/biocore/qiita/badge.png?branch=master)](https://coveralls.io/r/biocore/qiita)

Qiita (canonically pronounced *cheetah*) is a meta-analysis environment for microbiome (and other "comparative -omics") datasets. You can find the public Qiita server at [qiita.microbio.me](http://qiita.microbio.me).
Qiita (canonically pronounced *cheetah*) is an analysis environment for microbiome (and other "comparative -omics") datasets. You can find the public Qiita server at [qiita.microbio.me](http://qiita.microbio.me).

Qiita is currently in pre-alpha status. We are very open to community contributions and feedback. If you're interested in contributing to Qiita, see [CONTRIBUTING.md](https://github.com/biocore/qiita/blob/master/CONTRIBUTING.md). If you'd like to report bugs or request features, you can do that in the [Qiita issue tracker](https://github.com/biocore/qiita/issues).

Expand Down
3 changes: 3 additions & 0 deletions qiita_db/analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -777,6 +777,9 @@ def _build_mapping_file(self, samples, conn_handler=None):
merged_map.to_csv(mapping_fp, index_label='#SampleID',
na_rep='unknown', sep='\t')

self._add_file("%d_analysis_mapping.txt" % self._id,
"plain_text", conn_handler=conn_handler)

def _add_file(self, filename, filetype, data_type=None, conn_handler=None):
"""adds analysis item to database

Expand Down
6 changes: 3 additions & 3 deletions qiita_db/data.py
Original file line number Diff line number Diff line change
Expand Up @@ -1338,7 +1338,7 @@ def delete(cls, processed_data_id):
QiitaDBStatusError
If the processed data status is not sandbox
QiitaDBError
If the processed data has (meta)analyses
If the processed data has analyses
"""
if cls(processed_data_id).status != 'sandbox':
raise QiitaDBStatusError(
Expand All @@ -1354,8 +1354,8 @@ def delete(cls, processed_data_id):
if analyses:
raise QiitaDBError(
"Processed data %d cannot be removed because it is linked to "
"the following (meta)analysis: %s" % (processed_data_id,
', '.join(analyses)))
"the following analysis: %s" % (processed_data_id,
', '.join(analyses)))

# delete
queue = "delete_processed_data_%d" % processed_data_id
Expand Down
9 changes: 2 additions & 7 deletions qiita_db/metadata_template/base_metadata_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -1014,13 +1014,8 @@ def to_dataframe(self):
conn_handler = SQLConnectionHandler()
cols = sorted(get_table_cols(self._table_name(self._id), conn_handler))
# Get all metadata for the template
sql = """SELECT {0}
FROM qiita.{1}
JOIN qiita.{2} USING (sample_id)
WHERE {3} = %s""".format(", ".join(cols),
self._table,
self._table_name(self.id),
self._id_column)
sql = "SELECT {0} FROM qiita.{1}".format(", ".join(cols),
self._table_name(self.id))
meta = conn_handler.execute_fetchall(sql, (self._id,))

# Create the dataframe and clean it up a bit
Expand Down
4 changes: 2 additions & 2 deletions qiita_db/search.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,8 +343,8 @@ def _parse_study_search_string(self, searchstr,

sample_sql = ("SELECT ss.sample_id,%s "
"FROM qiita.study_sample ss "
"JOIN qiita.sample_{0} sa USING (sample_id) "
"JOIN qiita.study st USING (study_id) "
"JOIN qiita.sample_{0} sa ON ss.sample_id = sa.sample_id"
" JOIN qiita.study st ON st.study_id = ss.study_id "
"WHERE %s" %
(','.join(header_info), sql_where))
return study_sql, sample_sql, meta_header_type_lookup.keys()
Expand Down
27 changes: 27 additions & 0 deletions qiita_db/test/test_analysis.py
Original file line number Diff line number Diff line change
Expand Up @@ -386,6 +386,7 @@ def test_get_samples(self):
self.assertEqual(obs, exp)

def test_build_mapping_file(self):
new_id = get_count('qiita.filepath') + 1
samples = {1: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196']}
self.analysis._build_mapping_file(samples,
conn_handler=self.conn_handler)
Expand All @@ -401,13 +402,28 @@ def test_build_mapping_file(self):

assert_frame_equal(obs, exp)

sql = """SELECT * FROM qiita.filepath
WHERE filepath=%s ORDER BY filepath_id"""
obs = self.conn_handler.execute_fetchall(
sql, ("%d_analysis_mapping.txt" % self.analysis.id,))

exp = [[15, '1_analysis_mapping.txt', 9, '852952723', 1, 1],
[new_id, '1_analysis_mapping.txt', 9, '2349935429', 1, 1]]
self.assertEqual(obs, exp)

sql = """SELECT * FROM qiita.analysis_filepath
WHERE analysis_id=%s ORDER BY filepath_id"""
obs = self.conn_handler.execute_fetchall(sql, (self.analysis.id,))
exp = [[1L, 14L, 2L], [1L, 15L, None], [1L, new_id, None]]

def test_build_mapping_file_duplicate_samples(self):
samples = {1: ['1.SKB8.640193', '1.SKB8.640193', '1.SKD8.640184']}
with self.assertRaises(QiitaDBError):
self.analysis._build_mapping_file(samples,
conn_handler=self.conn_handler)

def test_build_biom_tables(self):
new_id = get_count('qiita.filepath') + 1
samples = {1: ['1.SKB8.640193', '1.SKD8.640184', '1.SKB7.640196']}
self.analysis._build_biom_tables(samples, 100,
conn_handler=self.conn_handler)
Expand All @@ -426,6 +442,17 @@ def test_build_biom_tables(self):
'Processed_id': 1}
self.assertEqual(obs, exp)

sql = """SELECT EXISTS(SELECT * FROM qiita.filepath
WHERE filepath_id=%s)"""
obs = self.conn_handler.execute_fetchone(sql, (new_id,))[0]

self.assertTrue(obs)

sql = """SELECT * FROM qiita.analysis_filepath
WHERE analysis_id=%s ORDER BY filepath_id"""
obs = self.conn_handler.execute_fetchall(sql, (self.analysis.id,))
exp = [[1L, 14L, 2L], [1L, 15L, None], [1L, new_id, None]]

def test_build_files(self):
self.analysis.build_files()

Expand Down
35 changes: 21 additions & 14 deletions qiita_db/test/test_search.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,9 @@ def test_parse_study_search_string(self):
"in ('integer', 'float8')")
exp_samp_sql = ("SELECT ss.sample_id,sa.altitude "
"FROM qiita.study_sample ss "
"JOIN qiita.sample_{0} sa USING (sample_id) "
"JOIN qiita.study st USING (study_id) "
"JOIN qiita.sample_{0} sa "
"ON ss.sample_id = sa.sample_id "
"JOIN qiita.study st ON st.study_id = ss.study_id "
"WHERE sa.altitude > 0")
self.assertEqual(st_sql, exp_st_sql)
self.assertEqual(samp_sql, exp_samp_sql)
Expand All @@ -44,8 +45,9 @@ def test_parse_study_search_string(self):
"in ('integer', 'float8')")
exp_samp_sql = ("SELECT ss.sample_id,sa.altitude "
"FROM qiita.study_sample ss "
"JOIN qiita.sample_{0} sa USING (sample_id) "
"JOIN qiita.study st USING (study_id) "
"JOIN qiita.sample_{0} sa "
"ON ss.sample_id = sa.sample_id "
"JOIN qiita.study st ON st.study_id = ss.study_id "
"WHERE NOT sa.altitude > 0")
self.assertEqual(st_sql, exp_st_sql)
self.assertEqual(samp_sql, exp_samp_sql)
Expand All @@ -59,8 +61,9 @@ def test_parse_study_search_string(self):
"('integer', 'float8')")
exp_samp_sql = ("SELECT ss.sample_id,sa.ph "
"FROM qiita.study_sample ss "
"JOIN qiita.sample_{0} sa USING (sample_id) "
"JOIN qiita.study st USING (study_id) "
"JOIN qiita.sample_{0} sa "
"ON ss.sample_id = sa.sample_id "
"JOIN qiita.study st ON st.study_id = ss.study_id "
"WHERE (sa.ph > 7 AND sa.ph < 9)")
self.assertEqual(st_sql, exp_st_sql)
self.assertEqual(samp_sql, exp_samp_sql)
Expand All @@ -74,8 +77,9 @@ def test_parse_study_search_string(self):
"('integer', 'float8')")
exp_samp_sql = ("SELECT ss.sample_id,sa.ph "
"FROM qiita.study_sample ss "
"JOIN qiita.sample_{0} sa USING (sample_id) "
"JOIN qiita.study st USING (study_id) "
"JOIN qiita.sample_{0} sa "
"ON ss.sample_id = sa.sample_id "
"JOIN qiita.study st ON st.study_id = ss.study_id "
"WHERE (sa.ph > 7 OR sa.ph < 9)")
self.assertEqual(st_sql, exp_st_sql)
self.assertEqual(samp_sql, exp_samp_sql)
Expand All @@ -90,8 +94,9 @@ def test_parse_study_search_string(self):
"and column_type in ('varchar')")
exp_samp_sql = ("SELECT ss.sample_id,sa.host_subject_id "
"FROM qiita.study_sample ss "
"JOIN qiita.sample_{0} sa USING (sample_id) "
"JOIN qiita.study st USING (study_id) "
"JOIN qiita.sample_{0} sa "
"ON ss.sample_id = sa.sample_id "
"JOIN qiita.study st ON st.study_id = ss.study_id "
"WHERE LOWER(sa.host_subject_id) "
"LIKE '%chicken little%'")
self.assertEqual(st_sql, exp_st_sql)
Expand All @@ -110,8 +115,9 @@ def test_parse_study_search_string(self):
exp_samp_sql = (
"SELECT ss.sample_id,sa.name "
"FROM qiita.study_sample ss "
"JOIN qiita.sample_{0} sa USING (sample_id) "
"JOIN qiita.study st USING (study_id) "
"JOIN qiita.sample_{0} sa "
"ON ss.sample_id = sa.sample_id "
"JOIN qiita.study st ON st.study_id = ss.study_id "
"WHERE (sa.name = 'Billy Bob' OR sa.name = 'Timmy' OR "
"(sa.name = 'Jimbo' AND sa.name > 25) OR sa.name < 5)")
self.assertEqual(st_sql, exp_st_sql)
Expand All @@ -131,8 +137,9 @@ def test_parse_study_search_string(self):
"lower('ph') and column_type in ('integer', 'float8')"]
exp_samp_sql = ("SELECT ss.sample_id,sa.pH,sa.ph "
"FROM qiita.study_sample ss "
"JOIN qiita.sample_{0} sa USING (sample_id) "
"JOIN qiita.study st USING (study_id) "
"JOIN qiita.sample_{0} sa "
"ON ss.sample_id = sa.sample_id "
"JOIN qiita.study st ON st.study_id = ss.study_id "
"WHERE (sa.ph > 7 OR sa.ph < 9)")
# use the split list to make sure the SQL is properly formed
self.assertEqual(len(st_sql), 2)
Expand Down
6 changes: 3 additions & 3 deletions qiita_pet/static/vendor/js/moi.js
Original file line number Diff line number Diff line change
Expand Up @@ -66,15 +66,15 @@ var moi = new function () {
* websocket. On construction, this method will send a message over the
* socket to get all known job information associated with this client.
*
* @param {host} The URL for the websocket, minus the ws:// header, or null
* to use the default moi-ws.
* @param {group_id} A group ID to get initial data from, or null to fetch
* all records associated with the user.
* @param {host} The URL for the websocket, minus the ws:// header, or null
* to use the default moi-ws.
* @param {on_open} Optional function for action when websocket is opened.
* @param {on_close} Optional function for action when websocket is closed.
* @param {on_error} Optional function for action when websocket errors.
*/
this.init = function(host, group_id, on_open, on_close, on_error) {
this.init = function(group_id, host, on_open, on_close, on_error) {
host = host || window.location.host + '/moi-ws/';
if (!("WebSocket" in window)) {
alert("Your browser does not appear to support websockets!");
Expand Down
4 changes: 2 additions & 2 deletions qiita_pet/templates/analysis_selected.html
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@
check_color();
}
$(document).ready(function() {
moi.init(window.location.host + '/analysis/selected/socket/', null, function(){}, error, error);
moi.init(null, window.location.host + '/analysis/selected/socket/', function(){}, error, error);
moi.add_callback('remove_pd', remove_pd_from_html);
moi.add_callback('remove_sample', remove_sample_from_html);
moi.add_callback('clear', clear_from_html);
Expand Down Expand Up @@ -149,7 +149,7 @@ <h4 class="modal-title" id="myModalLabel">Processed Data {{pid}}</h4>
<div class="modal-content">
<div class="modal-header">
<button type="button" class="close" data-dismiss="modal" aria-hidden="true">&times;</button>
<h4 class="modal-title" id="myModalLabel">Create new meta-analysis</h4>
<h4 class="modal-title" id="myModalLabel">Create new analysis</h4>
</div>
<form role="form" action="/analysis/3" method="post">
<input type="hidden" name="action" value="create">
Expand Down
2 changes: 1 addition & 1 deletion qiita_pet/templates/list_studies.html
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
}

$(document).ready(function() {
moi.init(window.location.host + '/study/list/socket/', null, function(){}, error, error);
moi.init(null, window.location.host + '/study/list/socket/', function(){}, error, error);
moi.add_callback('sel', show_alert);
function format ( d, row ) {
// `d` is the original data object for the row
Expand Down
6 changes: 3 additions & 3 deletions qiita_pet/templates/sitebase.html
Original file line number Diff line number Diff line change
Expand Up @@ -117,12 +117,12 @@
<img style="max-width:100px; margin-top: -7px;" src="/static/img/logo-clear.png">
</a>
</li>
<!-- meta-analysis menu -->
<!-- analysis menu -->
{% if user %}
<li class="dropdown">
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Meta Analysis<b class="caret"></b></a>
<a href="#" data-toggle="dropdown" class="dropdown-toggle">Analysis<b class="caret"></b></a>
<ul class="dropdown-menu">
<li><a href="/analysis/selected/">View Selected Samples</a></li>
<li><a href="/analysis/selected/">Create From Selected Samples</a></li>
<li><a href="/analysis/show/">See Previous Analyses</a></li>
</ul>
</li>
Expand Down
2 changes: 1 addition & 1 deletion qiita_ware/dispatchable.py
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ def submit_to_VAMPS(preprocessed_data_id):

def run_analysis(analysis_id, commands, comm_opts=None,
rarefaction_depth=None, **kwargs):
"""Run a meta-analysis"""
"""Run an analysis"""
analysis = Analysis(analysis_id)
ar = RunAnalysis(**kwargs)
return ar(analysis, commands, comm_opts, rarefaction_depth)
Expand Down