Skip to content

qiita.slurm_resource_allocations insertion #3415

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 21 commits into from
Jun 19, 2024
Merged

qiita.slurm_resource_allocations insertion #3415

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
88dc031
Update to DB qiita.slurm_resource_allocations
Gossty Apr 30, 2024
447e01a
Merge branch 'dev' of https://github.com/qiita-spots/qiita into dev
Gossty May 7, 2024
4578086
connected tests to database
Gossty May 8, 2024
e357512
Update util.py
Gossty May 8, 2024
42c1974
debugging changes to test
Gossty May 10, 2024
aa708a2
Update test_util.py
Gossty May 14, 2024
933bdde
Update test_util.py
Gossty May 15, 2024
8102d3b
Tests update
Gossty May 15, 2024
900d915
Update test_meta_util.py
Gossty May 16, 2024
3f70bc1
Updates to @antgonza comments
Gossty May 16, 2024
4fc8735
Updates to @charles-cowart comments
Gossty May 16, 2024
470d246
Resource Allocation Insertion into DB
Gossty Jun 5, 2024
3cb0e87
Merge branch 'dev' into test-util-db
Gossty Jun 5, 2024
f9d678d
Debug Memory Model
Gossty Jun 5, 2024
b77ef2e
Merge branch 'test-util-db' of https://github.com/Gossty/qiita into t…
Gossty Jun 5, 2024
022cd7c
Update test_util.py
Gossty Jun 5, 2024
beba0ca
Added columns to slurm_resource_allocation table
Gossty Jun 13, 2024
e004f2a
Update test_processing_job.py
Gossty Jun 13, 2024
61cb992
Update test_util.py
Gossty Jun 13, 2024
07fa018
Automatic date adjustment in utill.py
Gossty Jun 18, 2024
7c848bc
Update util.py
Gossty Jun 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Added columns to slurm_resource_allocation table
  • Loading branch information
@Gossty
Gossty committed Jun 13, 2024
commit beba0cad97e84700ea6194aa38b056db53b6ff10
3 changes: 3 additions & 0 deletions qiita_db/support_files/patches/92.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ CREATE TABLE qiita.slurm_resource_allocations (
extra_info varchar DEFAULT NULL,
memory_used bigint,
walltime_used integer,
start_stamp TIMESTAMP,
node_name varchar DEFAULT NULL,
node_model varchar DEFAULT NULL,
CONSTRAINT pk_slurm_resource_allocations_processing_job_id PRIMARY KEY (
processing_job_id )
);
Expand Down
922 changes: 461 additions & 461 deletions qiita_db/support_files/patches/test_db_sql/92.sql
View file
Open in desktop

Large diffs are not rendered by default.

20 changes: 10 additions & 10 deletions qiita_db/support_files/populate_test_db.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,16 +329,16 @@ INSERT INTO qiita.processing_job_status VALUES (6, 'waiting', 'The job is waitin
-- Data for Name: processing_job; Type: TABLE DATA; Schema: qiita; Owner: antoniog
--

INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('46b76f74-e100-47aa-9bf2-c0208bcea52d', 'test@foo.bar', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "True", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('80bf25f3-5f1d-4e10-9369-315e4244f6d5', 'test@foo.bar', 3, '{"reference": "2", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('9ba5ae7a-41e1-4202-b396-0259aeaac366', 'test@foo.bar', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('e5609746-a985-41a1-babf-6b3ebe9eb5a9', 'test@foo.bar', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f', 'test@foo.bar', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "False", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0', 'test@foo.bar', 12, '{"biom_table": "8", "depth": "9000", "subsample_multinomial": "False"}', 3, NULL, NULL, NULL, NULL, false, NULL);
INSERT INTO qiita.processing_job VALUES ('6d368e16-2242-4cf8-87b4-a5dc40bb890b', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1284411757);
INSERT INTO qiita.processing_job VALUES ('4c7115e8-4c8e-424c-bf25-96c292ca1931', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, NULL, NULL, NULL, false, 1287244546);
INSERT INTO qiita.processing_job VALUES ('3c9991ab-6c14-4368-a48c-841e8837a79c', 'test@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 3, NULL, NULL, NULL, NULL, false, 1284411377);
INSERT INTO qiita.processing_job VALUES ('b72369f9-a886-4193-8d3d-f7b504168e75', 'shared@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":true,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 3, NULL, '2015-11-22 21:15:00', NULL, NULL, false, 128552986);
INSERT INTO qiita.processing_job VALUES ('46b76f74-e100-47aa-9bf2-c0208bcea52d', 'test@foo.bar', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "True", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, 1279011391);
INSERT INTO qiita.processing_job VALUES ('80bf25f3-5f1d-4e10-9369-315e4244f6d5', 'test@foo.bar', 3, '{"reference": "2", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1286151876);
INSERT INTO qiita.processing_job VALUES ('9ba5ae7a-41e1-4202-b396-0259aeaac366', 'test@foo.bar', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1283300404);
INSERT INTO qiita.processing_job VALUES ('e5609746-a985-41a1-babf-6b3ebe9eb5a9', 'test@foo.bar', 3, '{"reference": "1", "similarity": "0.97", "sortmerna_e_value": "1", "sortmerna_max_pos": "10000", "input_data": "2", "threads": "1", "sortmerna_coverage": "0.97"}', 3, NULL, NULL, NULL, NULL, false, 1275827198);
INSERT INTO qiita.processing_job VALUES ('6ad4d590-4fa3-44d3-9a8f-ddbb472b1b5f', 'test@foo.bar', 1, '{"max_barcode_errors": "1.5", "sequence_max_n": "0", "max_bad_run_length": "3", "phred_offset": "auto", "rev_comp": "False", "phred_quality_threshold": "3", "input_data": "1", "rev_comp_barcode": "False", "rev_comp_mapping_barcodes": "False", "min_per_read_length_fraction": "0.75", "barcode_type": "golay_12"}', 3, NULL, NULL, NULL, NULL, false, 1266027);
INSERT INTO qiita.processing_job VALUES ('8a7a8461-e8a1-4b4e-a428-1bc2f4d3ebd0', 'test@foo.bar', 12, '{"biom_table": "8", "depth": "9000", "subsample_multinomial": "False"}', 3, NULL, NULL, NULL, NULL, false, 126652530);
INSERT INTO qiita.processing_job VALUES ('063e553b-327c-4818-ab4a-adfe58e49860', 'test@foo.bar', 1, '{"max_bad_run_length":3,"min_per_read_length_fraction":0.75,"sequence_max_n":0,"rev_comp_barcode":false,"rev_comp_mapping_barcodes":false,"rev_comp":false,"phred_quality_threshold":3,"barcode_type":"golay_12","max_barcode_errors":1.5,"input_data":1,"phred_offset":"auto"}', 1, NULL, NULL, NULL, NULL, true, NULL);
INSERT INTO qiita.processing_job VALUES ('bcc7ebcd-39c1-43e4-af2d-822e3589f14d', 'test@foo.bar', 2, '{"min_seq_len":100,"max_seq_len":1000,"trim_seq_length":false,"min_qual_score":25,"max_ambig":6,"max_homopolymer":6,"max_primer_mismatch":0,"barcode_type":"golay_12","max_barcode_errors":1.5,"disable_bc_correction":false,"qual_score_window":0,"disable_primers":false,"reverse_primers":"disable","reverse_primer_mismatches":0,"truncate_ambi_bases":false,"input_data":1}', 2, NULL, '2015-11-22 21:00:00', 'demultiplexing', NULL, true, NULL);
INSERT INTO qiita.processing_job VALUES ('d19f76ee-274e-4c1b-b3a2-a12d73507c55', 'shared@foo.bar', 3, '{"reference":1,"sortmerna_e_value":1,"sortmerna_max_pos":10000,"similarity":0.97,"sortmerna_coverage":0.97,"threads":1,"input_data":2}', 4, 1, '2015-11-22 21:30:00', 'generating demux file', NULL, true, NULL);
Expand Down
Binary file removed qiita_db/test/test_data/jobs_2024-02-21.tsv.gz
View file
Open in desktop
Binary file not shown.
Binary file added qiita_db/test/test_data/slurm_data.txt.gz
View file
Open in desktop
Binary file not shown.
35 changes: 4 additions & 31 deletions qiita_db/test/test_util.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1315,7 +1315,7 @@ def setUp(self):
self.columns = [
"sName", "sVersion", "cID", "cName", "processing_job_id",
"parameters", "samples", "columns", "input_size", "extra_info",
"MaxRSSRaw", "ElapsedRaw"]
"MaxRSSRaw", "ElapsedRaw", "Start", "node_name", "node_model"]

# df is a dataframe that represents a table with columns specified in
# self.columns
Expand Down Expand Up @@ -1397,35 +1397,8 @@ def test_MaxRSS_helper(self):
self.assertEqual(qdb.util.MaxRSS_helper(x), y)

def test_db_update(self):

def read_slurm(id):
sacct_dummies = {
1005932: (
"JobID|ElapsedRaw|MaxRSS|Submit|Start|MaxRSS|CPUTimeRAW|"
"ReqMem|AllocCPUS|AveVMSize|\n"
"1005932|165||2023-02-23T14:55:07|2023-02-23T14:55:08|"
"|165|120Gn|1||\n"
"1005932.batch|165|328716K|2023-02-23T14:55:08"
"|2023-02-23T14:55:08|328716K|"
"165|120Gn|1|156284K|\n"
"1005932.extern|165|0|2023-02-23T14:55:08|"
"2023-02-23T14:55:08|0|165|120Gn|1|108052K|"
),
1001100: (
"JobID|ElapsedRaw|MaxRSS|Submit|Start|MaxRSS|"
"CPUTimeRAW|ReqMem|AllocCPUS|AveVMSize|\n"
"1001100|219||2023-02-22T11:05:26|"
"2023-02-22T11:05:27||219|120Gn|1||\n"
"1001100.batch|219|342204K|2023-02-22T11:05:27|"
"2023-02-22T11:05:27|342204K|219|120Gn|1|156284K|\n"
"1001100.extern|219|0|2023-02-22T11:05:27|"
"2023-02-22T11:05:27|0|219|120Gn|1|108052K|"
)
}

slurm_info = sacct_dummies.get(id)
return slurm_info

path_to_data = './qiita_db/test/test_data/slurm_data.txt.gz'
test_data = pd.read_csv(path_to_data, sep="|")
types = {
'Split libraries FASTQ': [
'6d368e16-2242-4cf8-87b4-a5dc40bb890b',
Expand All @@ -1444,7 +1417,7 @@ def read_slurm(id):
]
}

qdb.util.update_resource_allocation_table(test=read_slurm)
qdb.util.update_resource_allocation_table(test=test_data)

for curr_cname, ids in types.items():
updated_df = qdb.util._retrieve_resource_data(
Expand Down
Loading