Skip to content

Solve stochastic failure #1046

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Apr 8, 2015
Merged

Solve stochastic failure #1046

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
da56fc5
Enforcing order in database cols
josenavas Apr 7, 2015
52505bc
Fixing prep template tests
josenavas Apr 7, 2015
e0b2698
Fixing sample template tests
josenavas Apr 7, 2015
6270a6d
Addressing @ElDeveloper's comment
josenavas Apr 8, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions qiita_db/metadata_template/base_metadata_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -542,7 +542,7 @@ def _add_common_creation_steps_to_queue(cls, md_template, obj_id,
headers = list(md_template.keys())

# Get the required columns from the DB
db_cols = get_table_cols(cls._table, conn_handler)
db_cols = sorted(get_table_cols(cls._table, conn_handler))
# Remove the sample_id and _id_column columns
db_cols.remove('sample_id')
db_cols.remove(cls._id_column)
Expand All @@ -561,7 +561,7 @@ def _add_common_creation_steps_to_queue(cls, md_template, obj_id,
values, many=True)

# Insert rows on *_columns table
headers = list(set(headers).difference(db_cols))
headers = sorted(set(headers).difference(db_cols))
datatypes = get_datatypes(md_template.ix[:, headers])
# psycopg2 requires a list of tuples, in which each tuple is a set
# of values to use in the string formatting of the query. We have all
Expand Down
106 changes: 56 additions & 50 deletions qiita_db/metadata_template/test/test_prep_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -582,39 +582,40 @@ def test_add_common_creation_steps_to_queue(self):
'VALUES (%s, %s, %s)')

sql_create_table = (
'CREATE TABLE qiita.prep_2 '
'(sample_id varchar NOT NULL, str_column varchar, '
'run_prefix varchar, barcodesequence varchar, platform varchar, '
'linkerprimersequence varchar, '
'experiment_design_description varchar, '
'library_construction_protocol varchar)')
'CREATE TABLE qiita.prep_2 (sample_id varchar NOT NULL, '
'barcodesequence varchar, experiment_design_description varchar, '
'library_construction_protocol varchar, '
'linkerprimersequence varchar, platform varchar, '
'run_prefix varchar, str_column varchar)')

sql_insert_dynamic = (
'INSERT INTO qiita.prep_2 '
'(sample_id, str_column, run_prefix, barcodesequence, platform, '
'linkerprimersequence, experiment_design_description, '
'library_construction_protocol) '
'(sample_id, barcodesequence, experiment_design_description, '
'library_construction_protocol, linkerprimersequence, platform, '
'run_prefix, str_column) '
'VALUES (%s, %s, %s, %s, %s, %s, %s, %s)')

sql_insert_dynamic_params_1 = (
'2.SKB8.640193', 'Value for sample 1', 's_G1_L001_sequences',
'GTCCGCAAGTTA', 'ILLUMINA', 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA')
'2.SKB8.640193', 'GTCCGCAAGTTA', 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
'Value for sample 1')
sql_insert_dynamic_params_2 = (
'2.SKD8.640184', 'Value for sample 2', 's_G1_L001_sequences',
'CGTAGAGCTCTC', 'ILLUMINA', 'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA')
'2.SKD8.640184', 'CGTAGAGCTCTC', 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
'Value for sample 2')

exp = [
(sql_insert_common, sql_insert_common_params_1),
(sql_insert_common, sql_insert_common_params_2),
(sql_insert_prep_columns, (2, 'str_column', 'varchar')),
(sql_insert_prep_columns, (2, 'run_prefix', 'varchar')),
(sql_insert_prep_columns, (2, 'barcodesequence', 'varchar')),
(sql_insert_prep_columns, (2, 'platform', 'varchar')),
(sql_insert_prep_columns, (2, 'linkerprimersequence', 'varchar')),
(sql_insert_prep_columns,
(2, 'experiment_design_description', 'varchar')),
(sql_insert_prep_columns,
(2, 'library_construction_protocol', 'varchar')),
(sql_insert_prep_columns, (2, 'linkerprimersequence', 'varchar')),
(sql_insert_prep_columns, (2, 'platform', 'varchar')),
(sql_insert_prep_columns, (2, 'run_prefix', 'varchar')),
(sql_insert_prep_columns, (2, 'str_column', 'varchar')),
(sql_create_table, None),
(sql_insert_dynamic, sql_insert_dynamic_params_1),
(sql_insert_dynamic, sql_insert_dynamic_params_2)]
Expand Down Expand Up @@ -794,17 +795,20 @@ def test_create(self):
# The new table hosts the correct values
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.prep_2")
# sample_id, study_id, str_column, ebi_submission_accession,
# run_prefix, barcodesequence, linkerprimersequence
exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA',
's_G1_L002_sequences', 'CCTCTGAGAGCT', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA',
's_G1_L001_sequences', 'GTCCGCAAGTTA', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA',
's_G1_L001_sequences', 'CGTAGAGCTCTC', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']]

# barcodesequence, ebi_submission_accession,
# experiment_design_description, library_construction_protocol,
# linkerprimersequence, platform, run_prefix, str_column
exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences',
'Value for sample 3'],
['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
'Value for sample 1'],
['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
'Value for sample 2']]

self.assertEqual(sorted(obs), sorted(exp))

# prep and qiime files have been created
Expand Down Expand Up @@ -858,17 +862,18 @@ def test_create_already_prefixed_samples(self):
# The new table hosts the correct values
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.prep_2")
# sample_id, study_id, str_column, ebi_submission_accession,
# run_prefix, barcodesequence, linkerprimersequence
exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA',
's_G1_L002_sequences', 'CCTCTGAGAGCT', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA',
's_G1_L001_sequences', 'GTCCGCAAGTTA', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA',
's_G1_L001_sequences', 'CGTAGAGCTCTC', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']]
# barcodesequence, ebi_submission_accession,
# experiment_design_description, library_construction_protocol,
# linkerprimersequence, platform, run_prefix, str_column
exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences',
'Value for sample 3'],
['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
'Value for sample 1'],
['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
'Value for sample 2']]
self.assertEqual(sorted(obs), sorted(exp))

# prep and qiime files have been created
Expand Down Expand Up @@ -959,17 +964,18 @@ def test_create_data_type_id(self):
# The new table hosts the correct values
obs = self.conn_handler.execute_fetchall(
"SELECT * FROM qiita.prep_2")
# sample_id, str_column, ebi_submission_accession,
# run_prefix, barcodesequence, linkerprimersequence
exp = [['1.SKB7.640196', 'Value for sample 3', 'ILLUMINA',
's_G1_L002_sequences', 'CCTCTGAGAGCT', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
['1.SKB8.640193', 'Value for sample 1', 'ILLUMINA',
's_G1_L001_sequences', 'GTCCGCAAGTTA', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA'],
['1.SKD8.640184', 'Value for sample 2', 'ILLUMINA',
's_G1_L001_sequences', 'CGTAGAGCTCTC', None,
'GTGCCAGCMGCCGCGGTAA', 'BBBB', 'AAAA']]
# barcodesequence, ebi_submission_accession,
# experiment_design_description, library_construction_protocol,
# linkerprimersequence, platform, run_prefix, str_column
exp = [['1.SKB7.640196', 'CCTCTGAGAGCT', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L002_sequences',
'Value for sample 3'],
['1.SKB8.640193', 'GTCCGCAAGTTA', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
'Value for sample 1'],
['1.SKD8.640184', 'CGTAGAGCTCTC', None, 'BBBB', 'AAAA',
'GTGCCAGCMGCCGCGGTAA', 'ILLUMINA', 's_G1_L001_sequences',
'Value for sample 2']]
self.assertEqual(sorted(obs), sorted(exp))

def test_create_error(self):
Expand Down
23 changes: 10 additions & 13 deletions qiita_db/metadata_template/test/test_sample_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -802,23 +802,20 @@ def test_add_common_creation_steps_to_queue(self):

sql_insert_required = (
'INSERT INTO qiita.required_sample_info '
'(study_id, sample_id, physical_location, has_physical_specimen, '
'has_extracted_data, sample_type, required_sample_info_status_id, '
'collection_timestamp, host_subject_id, description, latitude, '
'longitude) '
'(study_id, sample_id, collection_timestamp, description, '
'has_extracted_data, has_physical_specimen, host_subject_id, '
'latitude, longitude, physical_location, '
'required_sample_info_status_id, sample_type) '
'VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)')
sql_insert_required_params_1 = (
2, '2.Sample1', 'location1', True, True, 'type1', 1,
datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified',
'Test Sample 1', 42.42, 41.41)
2, '2.Sample1', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 1',
True, True, 'NotIdentified', 42.42, 41.41, 'location1', 1, 'type1')
sql_insert_required_params_2 = (
2, '2.Sample2', 'location1', True, True, 'type1', 1,
datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified',
'Test Sample 2', 4.2, 1.1)
2, '2.Sample2', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 2',
True, True, 'NotIdentified', 4.2, 1.1, 'location1', 1, 'type1')
sql_insert_required_params_3 = (
2, '2.Sample3', 'location1', True, True, 'type1', 1,
datetime(2014, 5, 29, 12, 24, 51), 'NotIdentified',
'Test Sample 3', 4.8, 4.41)
2, '2.Sample3', datetime(2014, 5, 29, 12, 24, 51), 'Test Sample 3',
True, True, 'NotIdentified', 4.8, 4.41, 'location1', 1, 'type1')

sql_insert_sample_cols = (
'INSERT INTO qiita.study_sample_columns '
Expand Down