Skip to content

Commit 5df155c

Browse files
hanarshanars
committed
split configs
1 parent fffd383 commit 5df155c

File tree

2 files changed

+177
-139
lines changed

2 files changed

+177
-139
lines changed

‎seqr/views/apis/data_manager_api_tests.py‎

Lines changed: 139 additions & 137 deletions
Original file line numberDiff line numberDiff line change
@@ -461,142 +461,6 @@
461461
PIPELINE_RUNNER_HOST = 'http://pipeline-runner:6000'
462462
PIPELINE_RUNNER_URL = f'{PIPELINE_RUNNER_HOST}/loading_pipeline_enqueue'
463463

464-
RNA_DATA_TYPE_PARAMS = {
465-
'outlier': {
466-
'model_cls': RnaSeqOutlier,
467-
'data_type': 'E',
468-
'message_data_type': 'Expression Outlier',
469-
'header': ['sampleID', 'project', 'geneID', 'tissue', 'detail', 'pValue', 'padjust', 'zScore'],
470-
'optional_headers': ['detail'],
471-
'loaded_data_row': ['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000240361', 'muscle', 'detail1', 0.01, 0.001, -3.1],
472-
'no_existing_data': ['NA19678', '1kg project nåme with uniçøde', 'ENSG00000233750', 'muscle', 'detail1', 0.064, '0.0000057', 7.8],
473-
'duplicated_indiv_id_data': [
474-
['NA20870', 'Test Reprocessed Project', 'ENSG00000233750', 'muscle', 'detail1', 0.064, '0.0000057', 7.8],
475-
['NA20870', '1kg project nåme with uniçøde', 'ENSG00000240361', 'fibroblasts', 'detail2', 0.01, 0.13, -3.1],
476-
],
477-
'write_data': {
478-
'{"gene_id": "ENSG00000233750", "p_value": "0.064", "p_adjust": "0.0000057", "z_score": "7.8"}\n',
479-
'{"gene_id": "ENSG00000240361", "p_value": "0.01", "p_adjust": "0.13", "z_score": "-3.1"}\n'
480-
},
481-
'new_data': [
482-
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000240361', 'muscle', 'detail1', 0.01, 0.13, -3.1],
483-
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000240361', 'muscle', 'detail2', 0.01, 0.13, -3.1],
484-
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000233750', 'muscle', 'detail1', 0.064, '0.0000057', 7.8],
485-
['NA19675_D3', 'Test Reprocessed Project', 'ENSG00000233750', 'muscle', 'detail1', 0.064, '0.0000057', 7.8],
486-
['NA20888', 'Test Reprocessed Project', 'ENSG00000240361', 'muscle', '', 0.04, 0.112, 1.9],
487-
],
488-
'skipped_samples': 'NA19675_D3 (Test Reprocessed Project)',
489-
'sample_tissue_type': 'M',
490-
'num_parsed_samples': 3,
491-
'initial_model_count': 3,
492-
'parsed_file_data': RNA_OUTLIER_SAMPLE_DATA,
493-
'get_models_json': lambda models: list(models.values_list('gene_id', 'p_adjust', 'p_value', 'z_score')),
494-
'expected_models_json': [
495-
('ENSG00000240361', 0.13, 0.01, -3.1), ('ENSG00000233750', 0.0000057, 0.064, 7.8),
496-
],
497-
'sample_guid': RNA_OUTLIER_MUSCLE_SAMPLE_GUID,
498-
},
499-
'tpm': {
500-
'model_cls': RnaSeqTpm,
501-
'data_type': 'T',
502-
'message_data_type': 'Expression',
503-
'header': ['sample_id', 'project', 'gene_id', 'individual_id', 'tissue', 'TPM'],
504-
'optional_headers': ['individual_id'],
505-
'loaded_data_row': ['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000135953', 'NA19675_D3', 'muscle', 1.34],
506-
'no_existing_data': ['NA19678', '1kg project nåme with uniçøde', 'ENSG00000233750', 'NA19678', 'muscle', 0.064],
507-
'duplicated_indiv_id_data': [
508-
['NA20870', 'Test Reprocessed Project', 'ENSG00000240361', 'NA20870', 'muscle', 7.8],
509-
['NA20870', '1kg project nåme with uniçøde', 'ENSG00000233750', 'NA20870', 'fibroblasts', 0.0],
510-
],
511-
'write_data': {'{"gene_id": "ENSG00000240361", "tpm": "7.8"}\n',
512-
'{"gene_id": "ENSG00000233750", "tpm": "0.0"}\n'},
513-
'new_data': [
514-
# existing sample NA19675_D2
515-
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000240361', 'NA19675_D2', 'muscle', 7.8],
516-
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000233750', 'NA19675_D2', 'muscle', 0.0],
517-
# no matched individual NA19675_D3
518-
['NA19675_D3', '1kg project nåme with uniçøde', 'ENSG00000233750', 'NA19675_D3', 'fibroblasts', 0.064],
519-
# a different project sample NA20888
520-
['NA20888', 'Test Reprocessed Project', 'ENSG00000240361', 'NA20888', 'muscle', 0.112],
521-
# a project mismatched sample NA20878
522-
['NA20878', 'Test Reprocessed Project', 'ENSG00000233750', 'NA20878', 'fibroblasts', 0.064],
523-
],
524-
'skipped_samples': 'NA19675_D3 (1kg project nåme with uniçøde), NA20878 (Test Reprocessed Project)',
525-
'sample_tissue_type': 'M',
526-
'num_parsed_samples': 4,
527-
'initial_model_count': 4,
528-
'deleted_count': 3,
529-
'parsed_file_data': RNA_TPM_SAMPLE_DATA,
530-
'get_models_json': lambda models: list(models.values_list('gene_id', 'tpm')),
531-
'expected_models_json': [('ENSG00000240361', 7.8), ('ENSG00000233750', 0.0)],
532-
'sample_guid': RNA_TPM_MUSCLE_SAMPLE_GUID,
533-
'mismatch_field': 'tpm',
534-
},
535-
'splice_outlier': {
536-
'model_cls': RnaSeqSpliceOutlier,
537-
'data_type': 'S',
538-
'message_data_type': 'Splice Outlier',
539-
'header': ['sampleID', 'projectName', 'geneID', 'chrom', 'start', 'end', 'strand', 'type', 'pValue', 'pAdjust',
540-
'deltaIntronJaccardIndex', 'counts', 'meanCounts', 'totalCounts', 'meanTotalCounts', 'tissue', 'rareDiseaseSamplesWithThisJunction',
541-
'rareDiseaseSamplesTotal'],
542-
'optional_headers': [],
543-
'loaded_data_row': ['NA19675_1', '1kg project nåme with uniçøde', 'ENSG00000240361', 'chr7', 132885746, 132886973, '*',
544-
'psi5', 1.08E-56, 3.08E-56, 12.34, 1297, 197, 129, 1297, 'fibroblasts', 0.53953638, 1, 20],
545-
'no_existing_data': ['NA19678', '1kg project nåme with uniçøde', 'ENSG00000240361', 'chr7', 132885746, 132886973, '*',
546-
'psi5', 1.08E-56, 3.08E-56, 12.34, 1297, 197, 129, 1297, 'fibroblasts', 0.53953638, 1, 20],
547-
'duplicated_indiv_id_data': [
548-
['NA20870', 'Test Reprocessed Project', 'ENSG00000233750', 'chr2', 167258096, 167258349, '*',
549-
'psi3', 1.56E-25, 6.33, 0.45, 143, 143, 143, 143, 'fibroblasts', 1, 20],
550-
['NA20870', '1kg project nåme with uniçøde', 'ENSG00000135953', 'chr2', 167258096, 167258349, '*',
551-
'psi3', 1.56E-25, 6.33, 0.45, 143, 143, 143, 143, 'muscle', 1, 20],
552-
],
553-
'write_data': {'{"chrom": "chr2", "start": "167258096",'
554-
' "end": "167258349", "strand": "*", "type": "psi3", "p_value": "1.56e-25", "p_adjust": "6.33",'
555-
' "delta_intron_jaccard_index": "0.45", "counts": "143",'
556-
' "mean_counts": "143", "total_counts": "143", "mean_total_counts": "143",'
557-
' "rare_disease_samples_with_this_junction": "1", "rare_disease_samples_total": "20", "gene_id": "ENSG00000233750"}\n',
558-
'{"chrom": "chr2", "start": "167258096",'
559-
' "end": "167258349", "strand": "*", "type": "psi3", "p_value": "1.56e-25", "p_adjust": "6.33",'
560-
' "delta_intron_jaccard_index": "0.45", "counts": "143",'
561-
' "mean_counts": "143", "total_counts": "143", "mean_total_counts": "143",'
562-
' "rare_disease_samples_with_this_junction": "1", "rare_disease_samples_total": "20", "gene_id": "ENSG00000135953"}\n',
563-
},
564-
'new_data': [
565-
# existing sample NA19675_1
566-
['NA19675_1', '1kg project nåme with uniçøde', 'ENSG00000233750;ENSG00000240361', 'chr2', 167254166, 167258349, '*', 'psi3',
567-
1.56E-25, -4.9, -0.46, 166, 16.6, 1660, 1.66, 'fibroblasts', 1, 20],
568-
['NA19675_1', '1kg project nåme with uniçøde', 'ENSG00000240361', 'chr7', 132885746, 132975168, '*', 'psi5',
569-
1.08E-56, -6.53, -0.85, 231, 0.231, 2313, 231.3, 'fibroblasts', 1, 20],
570-
# no matched individual NA19675_D3
571-
['NA19675_D3', '1kg project nåme with uniçøde', 'ENSG00000233750', 'chr2', 167258096, 167258349, '*',
572-
'psi3', 1.56E-25, 6.33, 0.45, 143, 14.3, 1433, 143.3, 'muscle', 1, 20],
573-
# a new sample NA20888
574-
['NA20888', 'Test Reprocessed Project', '', 'chr2', 167258096, 167258349, '*',
575-
'psi3', 1.56E-25, 6.33, 0.45, 143, 14.3, 1433, 143.3, 'fibroblasts', 1, 20],
576-
# a project mismatched sample NA20878
577-
['NA20878', 'Test Reprocessed Project', 'ENSG00000233750', 'chr2', 167258096, 167258349, '*', 'psi3',
578-
1.56E-25, 6.33, 0.45, 143, 14.3, 1433, 143.3, 'fibroblasts', 1, 20],
579-
],
580-
'skipped_samples': 'NA19675_D3 (1kg project nåme with uniçøde), NA20878 (Test Reprocessed Project)',
581-
'sample_tissue_type': 'F',
582-
'num_parsed_samples': 4,
583-
'initial_model_count': 7,
584-
'deleted_count': 4,
585-
'parsed_file_data': RNA_SPLICE_SAMPLE_DATA,
586-
'allow_missing_gene': True,
587-
'get_models_json': lambda models: list(
588-
models.values_list('gene_id', 'chrom', 'start', 'end', 'strand', 'type', 'p_value', 'p_adjust', 'delta_intron_jaccard_index',
589-
'counts', 'rare_disease_samples_with_this_junction', 'rare_disease_samples_total')),
590-
'expected_models_json': [
591-
('ENSG00000233750', '2', 167254166, 167258349, '*', 'psi3', 1.56e-25, -4.9, -0.46, 166, 1, 20),
592-
('ENSG00000240361', '2', 167254166, 167258349, '*', 'psi3', 1.56e-25, -4.9, -0.46, 166, 1, 20),
593-
('ENSG00000240361', '7', 132885746, 132975168, '*', 'psi5', 1.08e-56, -6.53, -0.85, 231, 1, 20)
594-
],
595-
'sample_guid': RNA_SPLICE_SAMPLE_GUID,
596-
'row_id': 'ENSG00000233750-2-167254166-167258349-*-psi3',
597-
},
598-
}
599-
600464

601465
@mock.patch('seqr.views.apis.data_manager_api.LOADING_DATASETS_DIR', '/local_datasets')
602466
@mock.patch('seqr.views.utils.permissions_utils.PM_USER_GROUP', 'project-managers')
@@ -753,6 +617,144 @@ def test_luigi_proxy(self, mock_hostname):
753617
responses.calls.reset()
754618
self._test_request_proxy('pipeline-runner-ui:8082', url, proxy_path='/api/task_list')
755619

620+
RNA_DATA_TYPE_PARAMS = {
621+
'outlier': {
622+
'model_cls': RnaSeqOutlier,
623+
'data_type': 'E',
624+
'message_data_type': 'Expression Outlier',
625+
'header': ['sampleID', 'project', 'geneID', 'tissue', 'detail', 'pValue', 'padjust', 'zScore'],
626+
'optional_headers': ['detail'],
627+
'loaded_data_row': ['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000240361', 'muscle', 'detail1',
628+
0.01, 0.001, -3.1],
629+
'no_existing_data': ['NA19678', '1kg project nåme with uniçøde', 'ENSG00000233750', 'muscle', 'detail1',
630+
0.064, '0.0000057', 7.8],
631+
'duplicated_indiv_id_data': [
632+
['NA20870', 'Test Reprocessed Project', 'ENSG00000233750', 'muscle', 'detail1', 0.064, '0.0000057',
633+
7.8],
634+
['NA20870', '1kg project nåme with uniçøde', 'ENSG00000240361', 'fibroblasts', 'detail2', 0.01, 0.13,
635+
-3.1],
636+
],
637+
'write_data': {
638+
'{"gene_id": "ENSG00000233750", "p_value": "0.064", "p_adjust": "0.0000057", "z_score": "7.8"}\n',
639+
'{"gene_id": "ENSG00000240361", "p_value": "0.01", "p_adjust": "0.13", "z_score": "-3.1"}\n'
640+
},
641+
'new_data': [
642+
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000240361', 'muscle', 'detail1', 0.01, 0.13,
643+
-3.1],
644+
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000240361', 'muscle', 'detail2', 0.01, 0.13,
645+
-3.1],
646+
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000233750', 'muscle', 'detail1', 0.064,
647+
'0.0000057', 7.8],
648+
['NA19675_D3', 'Test Reprocessed Project', 'ENSG00000233750', 'muscle', 'detail1', 0.064, '0.0000057',
649+
7.8],
650+
['NA20888', 'Test Reprocessed Project', 'ENSG00000240361', 'muscle', '', 0.04, 0.112, 1.9],
651+
],
652+
'skipped_samples': 'NA19675_D3 (Test Reprocessed Project)',
653+
'sample_tissue_type': 'M',
654+
'num_parsed_samples': 3,
655+
'initial_model_count': 3,
656+
'parsed_file_data': RNA_OUTLIER_SAMPLE_DATA,
657+
'sample_guid': RNA_OUTLIER_MUSCLE_SAMPLE_GUID,
658+
},
659+
'tpm': {
660+
'model_cls': RnaSeqTpm,
661+
'data_type': 'T',
662+
'message_data_type': 'Expression',
663+
'header': ['sample_id', 'project', 'gene_id', 'individual_id', 'tissue', 'TPM'],
664+
'optional_headers': ['individual_id'],
665+
'loaded_data_row': ['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000135953', 'NA19675_D3',
666+
'muscle', 1.34],
667+
'no_existing_data': ['NA19678', '1kg project nåme with uniçøde', 'ENSG00000233750', 'NA19678', 'muscle',
668+
0.064],
669+
'duplicated_indiv_id_data': [
670+
['NA20870', 'Test Reprocessed Project', 'ENSG00000240361', 'NA20870', 'muscle', 7.8],
671+
['NA20870', '1kg project nåme with uniçøde', 'ENSG00000233750', 'NA20870', 'fibroblasts', 0.0],
672+
],
673+
'write_data': {'{"gene_id": "ENSG00000240361", "tpm": "7.8"}\n',
674+
'{"gene_id": "ENSG00000233750", "tpm": "0.0"}\n'},
675+
'new_data': [
676+
# existing sample NA19675_D2
677+
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000240361', 'NA19675_D2', 'muscle', 7.8],
678+
['NA19675_D2', '1kg project nåme with uniçøde', 'ENSG00000233750', 'NA19675_D2', 'muscle', 0.0],
679+
# no matched individual NA19675_D3
680+
['NA19675_D3', '1kg project nåme with uniçøde', 'ENSG00000233750', 'NA19675_D3', 'fibroblasts', 0.064],
681+
# a different project sample NA20888
682+
['NA20888', 'Test Reprocessed Project', 'ENSG00000240361', 'NA20888', 'muscle', 0.112],
683+
# a project mismatched sample NA20878
684+
['NA20878', 'Test Reprocessed Project', 'ENSG00000233750', 'NA20878', 'fibroblasts', 0.064],
685+
],
686+
'skipped_samples': 'NA19675_D3 (1kg project nåme with uniçøde), NA20878 (Test Reprocessed Project)',
687+
'sample_tissue_type': 'M',
688+
'num_parsed_samples': 4,
689+
'initial_model_count': 4,
690+
'deleted_count': 3,
691+
'parsed_file_data': RNA_TPM_SAMPLE_DATA,
692+
'sample_guid': RNA_TPM_MUSCLE_SAMPLE_GUID,
693+
},
694+
'splice_outlier': {
695+
'model_cls': RnaSeqSpliceOutlier,
696+
'data_type': 'S',
697+
'message_data_type': 'Splice Outlier',
698+
'header': ['sampleID', 'projectName', 'geneID', 'chrom', 'start', 'end', 'strand', 'type', 'pValue',
699+
'pAdjust',
700+
'deltaIntronJaccardIndex', 'counts', 'meanCounts', 'totalCounts', 'meanTotalCounts', 'tissue',
701+
'rareDiseaseSamplesWithThisJunction',
702+
'rareDiseaseSamplesTotal'],
703+
'optional_headers': [],
704+
'loaded_data_row': ['NA19675_1', '1kg project nåme with uniçøde', 'ENSG00000240361', 'chr7', 132885746,
705+
132886973, '*',
706+
'psi5', 1.08E-56, 3.08E-56, 12.34, 1297, 197, 129, 1297, 'fibroblasts', 0.53953638, 1,
707+
20],
708+
'no_existing_data': ['NA19678', '1kg project nåme with uniçøde', 'ENSG00000240361', 'chr7', 132885746,
709+
132886973, '*',
710+
'psi5', 1.08E-56, 3.08E-56, 12.34, 1297, 197, 129, 1297, 'fibroblasts', 0.53953638, 1,
711+
20],
712+
'duplicated_indiv_id_data': [
713+
['NA20870', 'Test Reprocessed Project', 'ENSG00000233750', 'chr2', 167258096, 167258349, '*',
714+
'psi3', 1.56E-25, 6.33, 0.45, 143, 143, 143, 143, 'fibroblasts', 1, 20],
715+
['NA20870', '1kg project nåme with uniçøde', 'ENSG00000135953', 'chr2', 167258096, 167258349, '*',
716+
'psi3', 1.56E-25, 6.33, 0.45, 143, 143, 143, 143, 'muscle', 1, 20],
717+
],
718+
'write_data': {'{"chrom": "chr2", "start": "167258096",'
719+
' "end": "167258349", "strand": "*", "type": "psi3", "p_value": "1.56e-25", "p_adjust": "6.33",'
720+
' "delta_intron_jaccard_index": "0.45", "counts": "143",'
721+
' "mean_counts": "143", "total_counts": "143", "mean_total_counts": "143",'
722+
' "rare_disease_samples_with_this_junction": "1", "rare_disease_samples_total": "20", "gene_id": "ENSG00000233750"}\n',
723+
'{"chrom": "chr2", "start": "167258096",'
724+
' "end": "167258349", "strand": "*", "type": "psi3", "p_value": "1.56e-25", "p_adjust": "6.33",'
725+
' "delta_intron_jaccard_index": "0.45", "counts": "143",'
726+
' "mean_counts": "143", "total_counts": "143", "mean_total_counts": "143",'
727+
' "rare_disease_samples_with_this_junction": "1", "rare_disease_samples_total": "20", "gene_id": "ENSG00000135953"}\n',
728+
},
729+
'new_data': [
730+
# existing sample NA19675_1
731+
['NA19675_1', '1kg project nåme with uniçøde', 'ENSG00000233750;ENSG00000240361', 'chr2', 167254166,
732+
167258349, '*', 'psi3',
733+
1.56E-25, -4.9, -0.46, 166, 16.6, 1660, 1.66, 'fibroblasts', 1, 20],
734+
['NA19675_1', '1kg project nåme with uniçøde', 'ENSG00000240361', 'chr7', 132885746, 132975168, '*',
735+
'psi5',
736+
1.08E-56, -6.53, -0.85, 231, 0.231, 2313, 231.3, 'fibroblasts', 1, 20],
737+
# no matched individual NA19675_D3
738+
['NA19675_D3', '1kg project nåme with uniçøde', 'ENSG00000233750', 'chr2', 167258096, 167258349, '*',
739+
'psi3', 1.56E-25, 6.33, 0.45, 143, 14.3, 1433, 143.3, 'muscle', 1, 20],
740+
# a new sample NA20888
741+
['NA20888', 'Test Reprocessed Project', '', 'chr2', 167258096, 167258349, '*',
742+
'psi3', 1.56E-25, 6.33, 0.45, 143, 14.3, 1433, 143.3, 'fibroblasts', 1, 20],
743+
# a project mismatched sample NA20878
744+
['NA20878', 'Test Reprocessed Project', 'ENSG00000233750', 'chr2', 167258096, 167258349, '*', 'psi3',
745+
1.56E-25, 6.33, 0.45, 143, 14.3, 1433, 143.3, 'fibroblasts', 1, 20],
746+
],
747+
'skipped_samples': 'NA19675_D3 (1kg project nåme with uniçøde), NA20878 (Test Reprocessed Project)',
748+
'sample_tissue_type': 'F',
749+
'num_parsed_samples': 4,
750+
'initial_model_count': 7,
751+
'deleted_count': 4,
752+
'parsed_file_data': RNA_SPLICE_SAMPLE_DATA,
753+
'allow_missing_gene': True,
754+
'sample_guid': RNA_SPLICE_SAMPLE_GUID,
755+
},
756+
}
757+
756758
def _has_expected_file_loading_logs(self, file, user, info=None, warnings=None, additional_logs=None, additional_logs_offset=None):
757759
expected_logs = [
758760
(f'==> gsutil ls {file}', None),
@@ -804,7 +806,7 @@ def _test_update_rna_seq(self, data_type, mock_open, mock_subprocess, mock_load_
804806
url = reverse(update_rna_seq)
805807
self.check_pm_login(url)
806808

807-
params = RNA_DATA_TYPE_PARAMS[data_type]
809+
params = self.RNA_DATA_TYPE_PARAMS[data_type]
808810
model_cls = params['model_cls']
809811
header = params['header']
810812
loaded_data_row = params['loaded_data_row']

0 commit comments

Comments
 (0)