Skip to content

Commit cca6de7

Browse files
GosstyGossty
authored
qiita.slurm_resource_allocations insertion (qiita-spots#3415)
* Update to DB qiita.slurm_resource_allocations * connected tests to database * Update util.py * debugging changes to test * Update test_util.py * Update test_util.py * Tests update * Update test_meta_util.py * Updates to @antgonza comments * Updates to @charles-cowart comments * Resource Allocation Insertion into DB - Added functionality of updating resource allocation data into db in qiita_db/util.py - Added tests for the added functionality in qiita_db/test/test_util.py - Moved MaxRSS_helper function from qiita_core/util.py to qiita_db/util.py. - Moved MaxRSS_helper test from qiita_core/tests/test_util.py to qiita_db/test/test_util.py * Debug Memory Model * Update test_util.py * Added columns to slurm_resource_allocation table * Update test_processing_job.py * Update test_util.py * Automatic date adjustment in utill.py * Update util.py
1 parent 9de3f38 commit cca6de7

File tree

12 files changed

+797
-507
lines changed

12 files changed

+797
-507
lines changed

‎notebooks/resource-allocation/generate-allocation-summary-arrays.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
from qiita_core.util import MaxRSS_helper
1+
from qiita_db.util import MaxRSS_helper
22
from qiita_db.software import Software
33
import datetime
44
from io import StringIO

‎notebooks/resource-allocation/generate-allocation-summary.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from json import loads
66
from os.path import join
77

8-
from qiita_core.util import MaxRSS_helper
8+
from qiita_db.util import MaxRSS_helper
99
from qiita_db.exceptions import QiitaDBUnknownIDError
1010
from qiita_db.processing_job import ProcessingJob
1111
from qiita_db.software import Software

‎qiita_core/tests/test_util.py

Lines changed: 1 addition & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010

1111
from qiita_core.util import (
1212
qiita_test_checker, execute_as_transaction, get_qiita_version,
13-
is_test_environment, get_release_info, MaxRSS_helper)
13+
is_test_environment, get_release_info)
1414
from qiita_db.meta_util import (
1515
generate_biom_and_metadata_release, generate_plugin_releases)
1616
import qiita_db as qdb
@@ -82,20 +82,6 @@ def test_get_release_info(self):
8282
self.assertEqual(biom_metadata_release, ('', '', ''))
8383
self.assertNotEqual(archive_release, ('', '', ''))
8484

85-
def test_MaxRSS_helper(self):
86-
tests = [
87-
('6', 6.0),
88-
('6K', 6000),
89-
('6M', 6000000),
90-
('6G', 6000000000),
91-
('6.9', 6.9),
92-
('6.9K', 6900),
93-
('6.9M', 6900000),
94-
('6.9G', 6900000000),
95-
]
96-
for x, y in tests:
97-
self.assertEqual(MaxRSS_helper(x), y)
98-
9985

10086
if __name__ == '__main__':
10187
main()

‎qiita_core/util.py

Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -151,15 +151,3 @@ def get_release_info(study_status='public'):
151151
archive_release = ((md5sum, filepath, timestamp))
152152

153153
return (biom_metadata_release, archive_release)
154-
155-
156-
def MaxRSS_helper(x):
157-
if x[-1] == 'K':
158-
y = float(x[:-1]) * 1000
159-
elif x[-1] == 'M':
160-
y = float(x[:-1]) * 1000000
161-
elif x[-1] == 'G':
162-
y = float(x[:-1]) * 1000000000
163-
else:
164-
y = float(x)
165-
return y

‎qiita_db/support_files/patches/92.sql

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,9 @@ CREATE TABLE qiita.slurm_resource_allocations (
1010
extra_info varchar DEFAULT NULL,
1111
memory_used bigint,
1212
walltime_used integer,
13+
job_start TIMESTAMP,
14+
node_name varchar DEFAULT NULL,
15+
node_model varchar DEFAULT NULL,
1316
CONSTRAINT pk_slurm_resource_allocations_processing_job_id PRIMARY KEY (
1417
processing_job_id )
1518
);

‎qiita_db/support_files/patches/test_db_sql/92.sql

Lines changed: 461 additions & 461 deletions
Large diffs are not rendered by default.

‎qiita_db/support_files/populate_test_db.sql

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -329,16 +329,16 @@ INSERT INTO qiita.processing_job_status VALUES (6, 'waiting', 'The job is waitin
329329
-- Data for Name: processing_job; Type: TABLE DATA; Schema: qiita; Owner: antoniog
330330
--
331331

332-
INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, NULL);
333-
INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, NULL);
334-
INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, NULL);
335-
INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, NULL);
336-
INSERT INTO qiita.processing_job VALUES ('46b76f74-e100-47aa-9bf2-c0208bcea52d', 'test@foo.bar', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "True", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, NULL);
337-
INSERT INTO qiita.processing_job VALUES ('80bf25f3-5f1d-4e10-9369-315e4244f6d5', 'test@foo.bar', 3, '{"reference": "2", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
338-
INSERT INTO qiita.processing_job VALUES ('9ba5ae7a-41e1-4202-b396-0259aeaac366', 'test@foo.bar', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
339-
INSERT INTO qiita.processing_job VALUES ('e5609746-a985-41a1-babf-6b3ebe9eb5a9', 'test@foo.bar', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
340-
INSERT INTO qiita.processing_job VALUES ('6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f', 'test@foo.bar', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "False", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, NULL);
341-
INSERT INTO qiita.processing_job VALUES ('8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0', 'test@foo.bar', 12, '{"biom_table": "8", "depth": "9000", "subsample_multinomial": "False"}', 3, NULL, NULL, NULL, NULL, false, NULL);
332+
INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757);
333+
INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1287244546);
334+
INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, 1284411377);
335+
INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, 128552986);
336+
INSERT INTO qiita.processing_job VALUES ('46b76f74-e100-47aa-9bf2-c0208bcea52d', 'test@foo.bar', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "True", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, 1279011391);
337+
INSERT INTO qiita.processing_job VALUES ('80bf25f3-5f1d-4e10-9369-315e4244f6d5', 'test@foo.bar', 3, '{"reference": "2", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1286151876);
338+
INSERT INTO qiita.processing_job VALUES ('9ba5ae7a-41e1-4202-b396-0259aeaac366', 'test@foo.bar', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1283300404);
339+
INSERT INTO qiita.processing_job VALUES ('e5609746-a985-41a1-babf-6b3ebe9eb5a9', 'test@foo.bar', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1275827198);
340+
INSERT INTO qiita.processing_job VALUES ('6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f', 'test@foo.bar', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "False", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, 1266027);
341+
INSERT INTO qiita.processing_job VALUES ('8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0', 'test@foo.bar', 12, '{"biom_table": "8", "depth": "9000", "subsample_multinomial": "False"}', 3, NULL, NULL, NULL, NULL, false, 126652530);
342342
INSERT INTO qiita.processing_job VALUES ('063e553b-327c-4818-ab4a-adfe58e49860', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 1, NULL, NULL, NULL, NULL, true, NULL);
343343
INSERT INTO qiita.processing_job VALUES ('bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'test@foo.bar', 2, '{"min_seq_len":100,"max_seq_len":1000,"trim_seq_length":false,"min_qual_score":25,"max_ambig":6,"max_homopolymer":6,"max_primer_mismatch":0,"barcode_type":"golay_12","max_barcode_errors":1.5,"disable_bc_correction":false,"qual_score_window":0,"disable_primers":false,"reverse_primers":"disable","reverse_primer_mismatches":0,"truncate_ambi_bases":false,"input_data":1}', 2, NULL, '2015-11-22 21:00:00', 'demultiplexing', NULL, true, NULL);
344344
INSERT INTO qiita.processing_job VALUES ('d19f76ee-274e-4c1b-b3a2-a12d73507c55', 'shared@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 4, 1, '2015-11-22 21:30:00', 'generating demux file', NULL, true, NULL);

‎qiita_db/test/test_data/jobs_2024-02-21.tsv.gz

-7.17 MB
Binary file not shown.

‎qiita_db/test/test_data/slurm_data.txt.gz

658 KB
Binary file not shown.

‎qiita_db/test/test_processing_job.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -970,7 +970,7 @@ def test_notification_mail_generation(self):
970970
pj = qdb.processing_job.ProcessingJob(jid)
971971
obs = pj._generate_notification_message('running', 'Yahoo!')
972972
exp = {'subject': ('Single Rarefaction: running, 8a7a8461-e8a1-'
973-
'4b4e-a428-1bc2f4d3ebd0 [Not Available]'),
973+
'4b4e-a428-1bc2f4d3ebd0 [126652530]'),
974974
'message': 'Analysis Job Single Rarefaction\n'
975975
f'{qiita_config.base_url}/analysis/description/1/\n'
976976
'New status: running'}

‎qiita_db/test/test_util.py

Lines changed: 65 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1316,7 +1316,7 @@ def setUp(self):
13161316
self.columns = [
13171317
"sName", "sVersion", "cID", "cName", "processing_job_id",
13181318
"parameters", "samples", "columns", "input_size", "extra_info",
1319-
"MaxRSSRaw", "ElapsedRaw"]
1319+
"MaxRSSRaw", "ElapsedRaw", "Start", "node_name", "node_model"]
13201320

13211321
# df is a dataframe that represents a table with columns specified in
13221322
# self.columns
@@ -1351,8 +1351,15 @@ def test_minimize_const(self):
13511351
failures_df = qdb.util._resource_allocation_failures(
13521352
self.df, k, a, b, bm, self.col_name, 'MaxRSSRaw')
13531353
failures = failures_df.shape[0]
1354-
self.assertEqual(bm, qdb.util.mem_model3, msg="""Best memory model
1355-
doesn't match""")
1354+
self.assertEqual(bm, qdb.util.mem_model3,
1355+
msg=f"""Best memory model
1356+
doesn't match
1357+
Coefficients:{k} {a} {b}
1358+
{qdb.util.mem_model1}, "qdb.util.mem_model1"
1359+
{qdb.util.mem_model2}, "qdb.util.mem_model2"
1360+
{qdb.util.mem_model3}, "qdb.util.mem_model3"
1361+
{qdb.util.mem_model4}, "qdb.util.mem_model4"
1362+
""")
13561363
self.assertEqual(failures, 0, "Number of failures must be 0")
13571364

13581365
# check that the algorithm chooses correct model for ElapsedRaw and
@@ -1365,10 +1372,63 @@ def test_minimize_const(self):
13651372
self.df, k, a, b, bm, self.col_name, 'ElapsedRaw')
13661373
failures = failures_df.shape[0]
13671374

1368-
self.assertEqual(bm, qdb.util.time_model1, msg="""Best time model
1369-
doesn't match""")
1375+
self.assertEqual(bm, qdb.util.time_model1,
1376+
msg=f"""Best time model
1377+
doesn't match
1378+
Coefficients:{k} {a} {b}
1379+
{qdb.util.time_model1}, "qdb.util.time_model1"
1380+
{qdb.util.time_model2}, "qdb.util.time_model2"
1381+
{qdb.util.time_model3}, "qdb.util.time_model3"
1382+
{qdb.util.time_model4}, "qdb.util.time_model4"
1383+
""")
13701384
self.assertEqual(failures, 1, "Number of failures must be 1")
13711385

1386+
def test_MaxRSS_helper(self):
1387+
tests = [
1388+
('6', 6.0),
1389+
('6K', 6000),
1390+
('6M', 6000000),
1391+
('6G', 6000000000),
1392+
('6.9', 6.9),
1393+
('6.9K', 6900),
1394+
('6.9M', 6900000),
1395+
('6.9G', 6900000000),
1396+
]
1397+
for x, y in tests:
1398+
self.assertEqual(qdb.util.MaxRSS_helper(x), y)
1399+
1400+
def test_db_update(self):
1401+
path_to_data = './qiita_db/test/test_data/slurm_data.txt.gz'
1402+
test_data = pd.read_csv(path_to_data, sep="|")
1403+
types = {
1404+
'Split libraries FASTQ': [
1405+
'6d368e16-2242-4cf8-87b4-a5dc40bb890b',
1406+
'4c7115e8-4c8e-424c-bf25-96c292ca1931',
1407+
'b72369f9-a886-4193-8d3d-f7b504168e75',
1408+
'46b76f74-e100-47aa-9bf2-c0208bcea52d',
1409+
'6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f'],
1410+
'Pick closed-reference OTUs': [
1411+
'3c9991ab-6c14-4368-a48c-841e8837a79c',
1412+
'80bf25f3-5f1d-4e10-9369-315e4244f6d5',
1413+
'9ba5ae7a-41e1-4202-b396-0259aeaac366',
1414+
'e5609746-a985-41a1-babf-6b3ebe9eb5a9',
1415+
],
1416+
'Single Rarefaction': [
1417+
'8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0'
1418+
]
1419+
}
1420+
1421+
qdb.util.update_resource_allocation_table(test=test_data)
1422+
1423+
for curr_cname, ids in types.items():
1424+
updated_df = qdb.util._retrieve_resource_data(
1425+
curr_cname, self.SNAME, self.columns)
1426+
updated_ids_set = set(updated_df['processing_job_id'])
1427+
previous_ids_set = set(self.df['processing_job_id'])
1428+
for id in ids:
1429+
self.assertTrue(id in updated_ids_set)
1430+
self.assertFalse(id in previous_ids_set)
1431+
13721432

13731433
STUDY_INFO = {
13741434
'study_id': 1,

0 commit comments

Comments
 (0)