Skip to content

Speedup SQL and software tests #1935

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Sep 19, 2016
Merged

Speedup SQL and software tests #1935

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
107088e
Fixing software tests and fixing inserting publications
josenavas Sep 16, 2016
9c92547
Fixing SQL tests
josenavas Sep 16, 2016
3301f6f
Merge branch 'speedup-tests' of https://github.com/biocore/qiita into…
josenavas Sep 16, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 13 additions & 4 deletions qiita_db/software.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,13 +561,22 @@ def add_publications(self, publications):
"""
with qdb.sql_connection.TRN:
sql = """INSERT INTO qiita.publication (doi, pubmed_id)
VALUES (%s, %s)"""
qdb.sql_connection.TRN.add(sql, publications, many=True)
SELECT %s, %s
WHERE NOT EXISTS(SELECT *
FROM qiita.publication
WHERE doi = %s)"""
args = [[doi, pid, doi] for doi, pid in publications]
qdb.sql_connection.TRN.add(sql, args, many=True)

sql = """INSERT INTO qiita.software_publication
(software_id, publication_doi)
VALUES (%s, %s)"""
sql_params = [[self.id, doi] for doi, _ in publications]
SELECT %s, %s
WHERE NOT EXISTS(SELECT *
FROM qiita.software_publication
WHERE software_id = %s AND
publication_doi = %s)"""
sql_params = [[self.id, doi, self.id, doi]
for doi, _ in publications]
qdb.sql_connection.TRN.add(sql, sql_params, many=True)
qdb.sql_connection.TRN.execute()

Expand Down
49 changes: 21 additions & 28 deletions qiita_db/test/test_software.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@
import qiita_db as qdb


class CommandTestsReadOnly(TestCase):
@qiita_test_checker()
class CommandTests(TestCase):
def setUp(self):
self.software = qdb.software.Software(1)
self.parameters = {
Expand Down Expand Up @@ -191,16 +192,6 @@ def test_outputs(self):
exp = [['OTU table', 'BIOM']]
self.assertEqual(obs, exp)


@qiita_test_checker()
class CommandTests(TestCase):
def setUp(self):
self.software = qdb.software.Software(1)
self.parameters = {
'req_param': ['string', None],
'opt_int_param': ['integer', '4'],
'opt_choice_param': ['choice:["opt1", "opt2"]', 'opt1']}

def test_create_error(self):
# no parameters
with self.assertRaises(qdb.exceptions.QiitaDBError):
Expand Down Expand Up @@ -256,7 +247,8 @@ def test_create(self):
self.assertEqual(obs.optional_parameters, exp_optional)


class SoftwareTestsReadOnly(TestCase):
@qiita_test_checker()
class SoftwareTests(TestCase):
def test_name(self):
self.assertEqual(qdb.software.Software(1).name, "QIIME")

Expand Down Expand Up @@ -297,9 +289,6 @@ def test_type(self):
self.assertEqual(qdb.software.Software(1).type,
"artifact transformation")


@qiita_test_checker()
class SoftwareTests(TestCase):
def test_create(self):
obs = qdb.software.Software.create(
"New Software", "0.1.0",
Expand All @@ -316,7 +305,8 @@ def test_create(self):
self.assertEqual(obs.type, 'artifact transformation')

# create with publications
exp_publications = [['10.1000/nmeth.f.101', '12345678']]
exp_publications = [['10.1000/nmeth.f.101', '12345678'],
['10.1001/nmeth.f.101', '23456789']]
obs = qdb.software.Software.create(
"Published Software", "1.0.0", "Another testing software",
"env_name", "start_plugin", "artifact transformation",
Expand All @@ -331,16 +321,22 @@ def test_create(self):
self.assertEqual(obs.type, 'artifact transformation')

def test_add_publications(self):
tester = qdb.software.Software(1)
self.assertEqual(tester.publications,
[['10.1038/nmeth.f.303', '20383131']])
tester.add_publications([['10.1000/nmeth.f.101', '12345678']])
exp = [['10.1038/nmeth.f.303', '20383131'],
['10.1000/nmeth.f.101', '12345678']]
self.assertItemsEqual(tester.publications, exp)
obs = qdb.software.Software.create(
"New Software", "0.1.0",
"This is adding a new software for testing", "env_name",
"start_plugin", "artifact transformation")
self.assertEqual(obs.publications, [])
obs.add_publications([['10.1000/nmeth.f.101', '12345678']])
exp = [['10.1000/nmeth.f.101', '12345678']]
self.assertItemsEqual(obs.publications, exp)

# Add a publication that already exists
obs.add_publications([['10.1000/nmeth.f.101', '12345678']])
self.assertItemsEqual(obs.publications, exp)

class DefaultParametersTestsReadOnly(TestCase):

@qiita_test_checker()
class DefaultParametersTests(TestCase):
def test_exists(self):
cmd = qdb.software.Command(1)
obs = qdb.software.DefaultParameters.exists(
Expand All @@ -352,7 +348,7 @@ def test_exists(self):
self.assertTrue(obs)

obs = qdb.software.DefaultParameters.exists(
cmd, max_bad_run_length=3, min_per_read_length_fraction=0.75,
cmd, max_bad_run_length=3, min_per_read_length_fraction=0.65,
sequence_max_n=0, rev_comp_barcode=False,
rev_comp_mapping_barcodes=False, rev_comp=False,
phred_quality_threshold=3, barcode_type="hamming_8",
Expand All @@ -375,9 +371,6 @@ def test_command(self):
self.assertEqual(
qdb.software.DefaultParameters(1).command, qdb.software.Command(1))


@qiita_test_checker()
class DefaultParametersTests(TestCase):
def test_create(self):
cmd = qdb.software.Command(1)
obs = qdb.software.DefaultParameters.create(
Expand Down
32 changes: 25 additions & 7 deletions qiita_db/test/test_sql.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,7 @@ def test_collection_job_trigger_bad_insert(self):
'VALUES (1, 3)')
obs = self.conn_handler.execute_fetchall(
'SELECT * FROM qiita.collection_job')
exp = [[1, 1]]
self.assertEqual(obs, exp)
self.assertNotIn([[1, 3]], obs)

def test_collection_job_trigger(self):
# make sure a correct job inserts successfully
Expand Down Expand Up @@ -102,6 +101,22 @@ def _create_root_artifact(self):
[afp for _, afp, _ in new_root.filepaths])
return new_root

def _create_child_artifact(self, parents):
"""Creates a new artifact with the given parents"""
# Add a child of 2 roots
fd, fp = mkstemp(suffix='_seqs.fna')
close(fd)
self._files_to_remove.append(fp)
with open(fp, 'w') as f:
f.write("test")
fp = [(fp, 4)]
params = qdb.software.Parameters.from_default_params(
qdb.software.DefaultParameters(1), {'input_data': 2})
new = qdb.artifact.Artifact.create(
fp, "Demultiplexed", parents=parents,
processing_parameters=params)
return new

def test_find_artifact_roots_is_root_without_children(self):
"""Correctly returns the root if the artifact is already the root
and doesn't have any children
Expand Down Expand Up @@ -154,12 +169,15 @@ def test_artifact_ancestry_leaf(self):
def test_artifact_ancestry_leaf_multiple_parents(self):
"""Correctly returns the ancestry of a leaf artifact w multiple parents
"""
sql = """INSERT INTO qiita.parent_artifact (artifact_id, parent_id)
VALUES (%s, %s)"""
self.conn_handler.execute(sql, [4, 3])
root = self._create_root_artifact()
parent1 = self._create_child_artifact([root])
parent2 = self._create_child_artifact([root])
child = self._create_child_artifact([parent1, parent2])

sql = "SELECT * FROM qiita.artifact_ancestry(%s)"
obs = self.conn_handler.execute_fetchall(sql, [4])
exp = [[4, 3], [3, 1], [4, 2], [2, 1]]
obs = self.conn_handler.execute_fetchall(sql, [child.id])
exp = [[child.id, parent1.id], [child.id, parent2.id],
[parent1.id, root.id], [parent2.id, root.id]]
self.assertItemsEqual(obs, exp)

def test_artifact_ancestry_middle(self):
Expand Down