Skip to content

Ebi step 3 #1464

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 13 commits into from
Sep 9, 2015
Merged

Ebi step 3 #1464

Show file tree
Hide file tree
Changes from 12 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
bc86472
changes to DB
antgonza Aug 31, 2015
8564165
removing not necessary test data
antgonza Sep 3, 2015
664d137
step 3, initial commit
antgonza Sep 3, 2015
6320220
merging with ebi-step-2
antgonza Sep 3, 2015
08c7b2b
figuringout error
antgonza Sep 3, 2015
40e2abf
fixing error in last commit and addressing some of @josenavas comments
antgonza Sep 3, 2015
9f05273
addressing @josenavas comments
antgonza Sep 4, 2015
3f938bf
fixing flake8
antgonza Sep 4, 2015
e813cf0
addressing @ElDeveloper and @mortonjt
antgonza Sep 8, 2015
31a9f7b
addressing @josenavas and %mortonjt comments
antgonza Sep 8, 2015
6286792
addressing @adamrp comments
antgonza Sep 8, 2015
47bb358
changing from _ to -, @adamrp
antgonza Sep 8, 2015
b21fec8
minor comment from @josenavas
antgonza Sep 9, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions qiita_db/metadata_template/test/test_sample_template.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,8 @@ def setUp(self):
'tot_nitro', 'samp_salinity', 'altitude',
'env_biome', 'country', 'ph', 'anonymized_name',
'tot_org_carb', 'description_duplicate',
'env_feature', 'latitude', 'longitude'}
'env_feature', 'latitude', 'longitude',
'scientific_name'}


class TestSampleReadOnly(BaseTestSample):
Expand Down Expand Up @@ -103,7 +104,7 @@ def test_get_categories(self):

def test_len(self):
"""Len returns the correct number of categories"""
self.assertEqual(len(self.tester), 29)
self.assertEqual(len(self.tester), 30)

def test_getitem_required(self):
"""Get item returns the correct metadata value from the required table
Expand Down Expand Up @@ -157,8 +158,8 @@ def test_values(self):
'ENVO:Temperate grasslands, savannas, and shrubland biome',
'GAZ:United States of America', 6.94, 'SKB8', 5,
'Burmese root', 'ENVO:plant-associated habitat', 74.0894932572,
65.3283470202}
self.assertEqual(set(obs), exp)
65.3283470202, '1118232'}
self.assertItemsEqual(set(obs), exp)

def test_items(self):
"""items returns an iterator over the (key, value) tuples"""
Expand All @@ -184,7 +185,8 @@ def test_items(self):
('description_duplicate', 'Burmese root'),
('env_feature', 'ENVO:plant-associated habitat'),
('latitude', 74.0894932572),
('longitude', 65.3283470202)}
('longitude', 65.3283470202),
('scientific_name', '1118232')}
self.assertEqual(set(obs), exp)

def test_get(self):
Expand Down Expand Up @@ -479,16 +481,16 @@ def setUp(self):

def test_metadata_headers(self):
obs = SampleTemplate.metadata_headers()
exp = {'physical_specimen_location', 'physical_specimen_remaining',
exp = ['physical_specimen_location', 'physical_specimen_remaining',
'dna_extracted', 'sample_type', 'collection_timestamp',
'host_subject_id', 'description', 'season_environment',
'assigned_from_geo', 'texture', 'taxon_id', 'depth',
'host_taxid', 'common_name', 'water_content_soil', 'elevation',
'temp', 'tot_nitro', 'samp_salinity', 'altitude', 'env_biome',
'country', 'ph', 'anonymized_name', 'tot_org_carb',
'description_duplicate', 'env_feature', 'latitude', 'longitude',
'sample_id'}
self.assertEqual(set(obs), exp)
'sample_id', 'scientific_name']
self.assertItemsEqual(obs, exp)

def test_study_id(self):
"""Ensure that the correct study ID is returned"""
Expand Down Expand Up @@ -543,7 +545,7 @@ def test_categories(self):
'physical_specimen_location',
'physical_specimen_remaining', 'dna_extracted',
'sample_type', 'collection_timestamp', 'host_subject_id',
'description', 'latitude', 'longitude'}
'description', 'latitude', 'longitude', 'scientific_name'}
obs = set(self.tester.categories())
self.assertItemsEqual(obs, exp)

Expand Down Expand Up @@ -1862,11 +1864,11 @@ def test_to_dataframe(self):
'water_content_soil', 'elevation', 'temp', 'tot_nitro',
'samp_salinity', 'altitude', 'env_biome', 'country', 'ph',
'anonymized_name', 'tot_org_carb', 'description_duplicate',
'env_feature'})
'env_feature', 'scientific_name'})

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of deleting this lines, can you just check that is returning an empty set? I think this was the original idea of this test but got lost when scientific_name become required. This way we will have a test for a template that is not missing anything.

def test_check_restrictions(self):
obs = self.tester.check_restrictions([SAMPLE_TEMPLATE_COLUMNS['EBI']])
self.assertEqual(obs, {'scientific_name'})
self.assertEqual(obs, set([]))

del self.metadata['collection_timestamp']
st = npt.assert_warns(QiitaDBWarning, SampleTemplate.create,
Expand Down
60 changes: 31 additions & 29 deletions qiita_db/support_files/populate_test_db.sql
View file
Open in desktop

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion qiita_db/test/test_setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ def test_required_sample_info(self):
self.assertEqual(get_count("qiita.study_sample"), 27)

def test_study_sample_columns(self):
self.assertEqual(get_count("qiita.study_sample_columns"), 30)
self.assertEqual(get_count("qiita.study_sample_columns"), 31)

def test_sample_1(self):
self.assertEqual(get_count("qiita.sample_1"), 27)
Expand Down
59 changes: 23 additions & 36 deletions qiita_ware/commands.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,8 @@
# The full license is in the file LICENSE, distributed with this software.
# -----------------------------------------------------------------------------

from os import makedirs
from os.path import join, isdir
from shutil import rmtree
from functools import partial
from tarfile import open as taropen
from tempfile import mkdtemp

Expand Down Expand Up @@ -43,13 +41,13 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
If fastq_dir_fp is passed, it must not contain any empty files, or
gzipped empty files
"""
# step 1
# step 1: init and validate
ebi_submission = EBISubmission(preprocessed_data_id, action)

# step 2
# step 2: generate demux fastq files
ebi_submission.preprocessed_data.update_insdc_status('demuxing samples')
try:
demux_samples = ebi_submission.generate_demultiplexed_fastq()
ebi_submission.generate_demultiplexed_fastq()
except:
error_msg = format_exc()
if isdir(ebi_submission.ebi_dir):
Expand All @@ -60,46 +58,35 @@ def submit_EBI(preprocessed_data_id, action, send, fastq_dir_fp=None):
info={'ebi_submission': preprocessed_data_id})
raise

# other steps
output_dir = fastq_dir_fp + '_submission'
sample_template = ebi_submission.sample_template
prep_template = ebi_submission.prep_template
preprocessed_data = ebi_submission.preprocessed_data

samp_fp = join(fastq_dir_fp, 'sample_metadata.txt')
prep_fp = join(fastq_dir_fp, 'prep_metadata.txt')

sample_template.to_file(samp_fp, demux_samples)
prep_template.to_file(prep_fp, demux_samples)

# Get specific output directory and set filepaths
get_output_fp = partial(join, output_dir)
study_fp = get_output_fp('study.xml')
sample_fp = get_output_fp('sample.xml')
experiment_fp = get_output_fp('experiment.xml')
run_fp = get_output_fp('run.xml')
submission_fp = get_output_fp('submission.xml')

if not isdir(output_dir):
makedirs(output_dir)
else:
raise IOError('The output folder already exists: %s' %
output_dir)

ebi_submission.write_all_xml_files(study_fp, sample_fp, experiment_fp,
run_fp, submission_fp, action)
# step 3: generate and write xml files
ebi_submission.write_xml_file(ebi_submission.generate_study_xml(),
'study_xml_fp',
ebi_submission.study_xml_fp)
ebi_submission.write_xml_file(ebi_submission.generate_sample_xml(),
'sample_xml_fp',
ebi_submission.sample_xml_fp)
ebi_submission.write_xml_file(ebi_submission.generate_experiment_xml(),
'experiment_xml_fp',
ebi_submission.experiment_xml_fp)
ebi_submission.write_xml_file(ebi_submission.generate_run_xml(),
'run_xml_fp',
ebi_submission.run_xml_fp)
ebi_submission.write_xml_file(ebi_submission.generate_submission_xml(),
'submission_xml_fp',
ebi_submission.submission_xml_fp)

# other steps
if send:
ebi_submission.send_sequences()
study_accession, submission_accession = ebi_submission.send_xml()

if study_accession is None or submission_accession is None:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Just to confirm, this block of code is already replicated somewhere else (in your previous PR IIRC), right?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, before we had different folders for different things and now we have 1 single folder.

preprocessed_data.update_insdc_status('failed')
ebi_submission.preprocessed_data.update_insdc_status('failed')

raise ComputeError("EBI Submission failed!")
else:
preprocessed_data.update_insdc_status('success', study_accession,
submission_accession)
ebi_submission.preprocessed_data.update_insdc_status(
'success', study_accession, submission_accession)
else:
study_accession, submission_accession = None, None

Expand Down
Loading