Merge pull request #5092 from broadinstitute/remove-rna-loading-manage-command

Remove rna loading manage command

Author: hanars

seqr/management/commands/load_rna_seq.py

@@ -1,7 +1,6 @@
 import base64
 from collections import defaultdict
 from datetime import datetime
-import gzip
 import json
 import os
 import requests
@@ -26,7 +25,7 @@
 from seqr.views.utils.airtable_utils import AirtableSession, LOADABLE_PDO_STATUSES, AVAILABLE_PDO_STATUS
 from seqr.views.utils.dataset_utils import load_rna_seq, load_phenotype_prioritization_data_file, RNA_DATA_TYPE_CONFIGS, \
     post_process_rna_data, convert_django_meta_to_http_headers
-from seqr.views.utils.file_utils import get_temp_file_path, load_uploaded_file, persist_temp_file
+from seqr.views.utils.file_utils import get_temp_file_path, load_uploaded_file
 from seqr.views.utils.json_utils import create_json_response
 from seqr.views.utils.json_to_orm_utils import update_model_from_json
 from seqr.views.utils.pedigree_info_utils import get_validated_related_individuals, JsonConstants
@@ -72,37 +71,15 @@ def update_rna_seq(request):
     if uploaded_mapping_file_id:
         mapping_file = load_uploaded_file(uploaded_mapping_file_id)
 
-    file_name_prefix = f'rna_sample_data__{data_type}__{datetime.now().isoformat()}'
-    file_dir = get_temp_file_path(file_name_prefix, is_local=True)
-    os.mkdir(file_dir)
-
-    sample_files = {}
-
-    def _save_sample_data(sample_key, sample_data):
-        if sample_key not in sample_files:
-            file_name = _get_sample_file_path(file_dir, '_'.join(sample_key))
-            sample_files[sample_key] = gzip.open(file_name, 'at')
-        sample_files[sample_key].write(f'{json.dumps(sample_data)}\n')
-
     try:
-        sample_guids_to_keys, info, warnings = load_rna_seq(
-            data_type, file_path, _save_sample_data,
+        sample_guids_to_keys, file_name_prefix, info, warnings = load_rna_seq(
+            data_type, file_path,
             user=request.user, mapping_file=mapping_file, ignore_extra_samples=request_json.get('ignoreExtraSamples'))
     except FileNotFoundError:
         return create_json_response({'error': 'File not found: {}'.format(file_path)}, status=400)
     except ValueError as e:
         return create_json_response({'error': str(e)}, status=400)
 
-    for sample_guid, sample_key in sample_guids_to_keys.items():
-        sample_files[sample_key].close()  # Required to ensure gzipped files are properly terminated
-        os.rename(
-            _get_sample_file_path(file_dir, '_'.join(sample_key)),
-            _get_sample_file_path(file_dir, sample_guid),
-        )
-
-    if sample_guids_to_keys:
-        persist_temp_file(file_name_prefix, request.user)
-
     return create_json_response({
         'info': info,
         'warnings': warnings,

seqr/management/tests/load_rna_seq_tests.py

@@ -793,12 +793,12 @@ def test_update_rna_splice_outlier(self, *args, **kwargs):
     @mock.patch('seqr.views.utils.file_utils.tempfile.gettempdir', lambda: 'tmp/')
     @mock.patch('seqr.utils.communication_utils.send_html_email')
     @mock.patch('seqr.utils.communication_utils.safe_post_to_slack')
-    @mock.patch('seqr.views.apis.data_manager_api.datetime')
-    @mock.patch('seqr.views.apis.data_manager_api.os.mkdir')
-    @mock.patch('seqr.views.apis.data_manager_api.os.rename')
+    @mock.patch('seqr.views.utils.dataset_utils.datetime')
+    @mock.patch('seqr.views.utils.dataset_utils.os.mkdir')
+    @mock.patch('seqr.views.utils.dataset_utils.os.rename')
     @mock.patch('seqr.views.apis.data_manager_api.load_uploaded_file')
     @mock.patch('seqr.utils.file_utils.subprocess.Popen')
-    @mock.patch('seqr.views.apis.data_manager_api.gzip.open')
+    @mock.patch('seqr.views.utils.dataset_utils.gzip.open')
     def _test_update_rna_seq(self, data_type, mock_open, mock_subprocess, mock_load_uploaded_file,
                             mock_rename, mock_mkdir, mock_datetime, mock_send_slack, mock_send_email):
         url = reverse(update_rna_seq)
@@ -906,7 +906,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s
                 f'Attempted data loading for {num_loaded_samples} RNA-seq samples in the following {num_projects}'
                 f' projects: {project_names}'
             ]
-            file_name = RNA_FILENAME_TEMPLATE.format(data_type)
+            file_name = RNA_FILENAME_TEMPLATE.format(params['data_type'])
             response_json = response.json()
             self.assertDictEqual(response_json, {'info': info, 'warnings': warnings or [], 'sampleGuids': mock.ANY,
                                                  'fileName': file_name})
@@ -974,7 +974,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s
         self.assertSetEqual(set(response_json['sampleGuids']), {sample_guid, new_sample_guid})
 
         # test correct file interactions
-        file_path = RNA_FILENAME_TEMPLATE.format(data_type)
+        file_path = RNA_FILENAME_TEMPLATE.format(params['data_type'])
         expected_subprocess_calls = [
             f'gsutil ls {RNA_FILE_ID}',
             f'gsutil cat {RNA_FILE_ID} | gunzip -c -q - ',
@@ -1027,7 +1027,7 @@ def _test_basic_data_loading(data, num_parsed_samples, num_loaded_samples, new_s
         self.assertTrue(second_tissue_sample_guid != new_sample_guid)
         self.assertTrue(second_tissue_sample_guid in response_json['sampleGuids'])
         self._assert_expected_file_open(mock_rename, mock_open, [
-            f'tmp/temp_uploads/{RNA_FILENAME_TEMPLATE.format(data_type)}/{sample_guid}.json.gz'
+            f'tmp/temp_uploads/{RNA_FILENAME_TEMPLATE.format(params["data_type"])}/{sample_guid}.json.gz'
             for sample_guid in response_json['sampleGuids']
         ])
         self.assertSetEqual(