Reduce ena submission induced db load

ena-submission/Snakefile

@@ -14,6 +14,10 @@ for key, value in defaults.items():
         config[key] = value
 
 LOG_LEVEL = config.get("log_level", "INFO")
+TIME_BETWEEN_ITERATIONS = config.get("time_between_iterations", 10)
+MIN_BETWEEN_GITHUB_REQUESTS = config.get("min_between_github_requests", 2)
+MIN_BETWEEN_ENA_CHECKS = config.get("min_between_ena_checks", 5)
+
 SUBMIT_TO_ENA_PROD = config.get("submit_to_ena_prod", False)
 if config.get("backend_url", "") not in config.get("allowed_submission_hosts", []):
     print("WARNING: backend_url not in allowed_hosts")
@@ -76,11 +80,13 @@ rule trigger_submission_to_ena:
         submitted=touch("results/triggered"),
     params:
         log_level=LOG_LEVEL,
+        min_between_github_requests=MIN_BETWEEN_GITHUB_REQUESTS,
     shell:
         """
         python {input.script} \
                 --config-file {input.config} \
                 --log-level {params.log_level} \
+                --min-between-github-requests {params.min_between_github_requests}
         """
 
 
@@ -111,11 +117,13 @@ rule create_project:
     params:
         log_level=LOG_LEVEL,
         test_flag="--test" if SUBMIT_TO_ENA_DEV else "",
+        time_between_iterations=TIME_BETWEEN_ITERATIONS,
     shell:
         """
         python {input.script} \
             --config-file {input.config} \
             --log-level {params.log_level} \
+            --time-between-iterations {params.time_between_iterations} \
             {params.test_flag}
         """
 
@@ -129,11 +137,13 @@ rule create_sample:
     params:
         log_level=LOG_LEVEL,
         test_flag="--test" if SUBMIT_TO_ENA_DEV else "",
+        time_between_iterations=TIME_BETWEEN_ITERATIONS,
     shell:
         """
         python {input.script} \
             --config-file {input.config} \
             --log-level {params.log_level} \
+            --time-between-iterations {params.time_between_iterations} \
             {params.test_flag}
         """
 
@@ -147,11 +157,15 @@ rule create_assembly:
     params:
         log_level=LOG_LEVEL,
         test_flag="--test" if SUBMIT_TO_ENA_DEV else "",
+        time_between_iterations=TIME_BETWEEN_ITERATIONS,
+        min_between_ena_checks=MIN_BETWEEN_ENA_CHECKS,
     shell:
         """
         python {input.script} \
             --config-file {input.config} \
             --log-level {params.log_level} \
+            --time-between-iterations {params.time_between_iterations} \
+            --min-between-ena-checks {params.min_between_ena_checks} \
             {params.test_flag}
         """
 
@@ -164,9 +178,11 @@ rule upload_to_loculus:
         sample_created=touch("results/uploaded_external_metadata"),
     params:
         log_level=LOG_LEVEL,
+        time_between_iterations=TIME_BETWEEN_ITERATIONS,
     shell:
         """
         python {input.script} \
             --config-file {input.config} \
             --log-level {params.log_level} \
+            --time-between-iterations {params.time_between_iterations} \
         """

ena-submission/config/defaults.yaml

@@ -9,6 +9,9 @@ ena_submission_password: fake-password
 submit_to_ena_prod: False
 allowed_submission_hosts:
   - https://backend.pathoplexus.org
+time_between_iterations: 10
+min_between_github_requests: 2
+min_between_ena_checks: 5
 #ena_checklist: ERC000033 - do not use until all fields are mapped to ENA accepted options
 metadata_mapping:
   'subject exposure':

ena-submission/scripts/create_assembly.py

@@ -543,7 +543,19 @@ def assembly_table_handle_errors(
     default=False,
     help="Allow multiple submissions of the same project for testing AND use the webin-cli test endpoint",
 )
-def create_assembly(log_level, config_file, test=False):
+@click.option(
+    "--time-between-iterations",
+    default=10,
+    type=int,
+)
+@click.option(
+    "--min-between-ena-checks",
+    default=5,
+    type=int,
+)
+def create_assembly(
+    log_level, config_file, test=False, time_between_iterations=10, min_between_ena_checks=5
+):
     logger.setLevel(log_level)
     logging.getLogger("requests").setLevel(logging.INFO)
 
@@ -565,9 +577,9 @@ def create_assembly(log_level, config_file, test=False):
         submission_table_update(db_config)
 
         assembly_table_create(db_config, config, retry_number=3, test=test)
-        assembly_table_update(db_config, config)
+        assembly_table_update(db_config, config, time_threshold=min_between_ena_checks)
         assembly_table_handle_errors(db_config, config, slack_config)
-        time.sleep(2)
+        time.sleep(time_between_iterations)
 
 
 if __name__ == "__main__":

ena-submission/scripts/create_project.py

@@ -272,7 +272,9 @@ def project_table_create(
                 )
             )
             continue
-        logger.info(f"Starting Project creation for group_id {row["group_id"]}")
+        logger.info(
+            f"Starting Project creation for group_id {row["group_id"]} organism {row["organism"]}"
+        )
         project_creation_results: CreationResults = create_ena_project(ena_config, project_set)
         if project_creation_results.results:
             update_values = {
@@ -296,7 +298,9 @@ def project_table_create(
                 )
                 tries += 1
             if number_rows_updated == 1:
-                logger.info(f"Project creation for group_id {row["group_id"]} succeeded!")
+                logger.info(
+                    f"Project creation for group_id {row["group_id"]} organism {row["organism"]} succeeded!"
+                )
         else:
             update_values = {
                 "status": Status.HAS_ERRORS,
@@ -370,7 +374,12 @@ def project_table_handle_errors(
     default=False,
     help="Allow multiple submissions of the same project for testing",
 )
-def create_project(log_level, config_file, test=False):
+@click.option(
+    "--time-between-iterations",
+    default=10,
+    type=int,
+)
+def create_project(log_level, config_file, test=False, time_between_iterations=10):
     logger.setLevel(log_level)
     logging.getLogger("requests").setLevel(logging.INFO)
 
@@ -393,7 +402,7 @@ def create_project(log_level, config_file, test=False):
 
         project_table_create(db_config, config, test=test)
         project_table_handle_errors(db_config, config, slack_config)
-        time.sleep(2)
+        time.sleep(time_between_iterations)
 
 
 if __name__ == "__main__":

ena-submission/scripts/create_sample.py

@@ -416,7 +416,12 @@ def sample_table_handle_errors(
     default=False,
     help="Allow multiple submissions of the same project for testing",
 )
-def create_sample(log_level, config_file, test=False):
+@click.option(
+    "--time-between-iterations",
+    default=10,
+    type=int,
+)
+def create_sample(log_level, config_file, test=False, time_between_iterations=10):
     logger.setLevel(log_level)
     logging.getLogger("requests").setLevel(logging.INFO)
 
@@ -439,7 +444,7 @@ def create_sample(log_level, config_file, test=False):
 
         sample_table_create(db_config, config, test=test)
         sample_table_handle_errors(db_config, config, slack_config)
-        time.sleep(2)
+        time.sleep(time_between_iterations)
 
 
 if __name__ == "__main__":

ena-submission/scripts/ena_submission_helper.py

@@ -128,13 +128,10 @@ def get_project_xml(project_set):
         }
 
     xml = get_project_xml(project_set)
-    try:
-        response = post_webin(config, xml)
-        response.raise_for_status()
-    except requests.exceptions.RequestException as e:
+    response = post_webin(config, xml)
+    if not response.ok:
         error_message = (
-            f"Request failed with status:{response.status_code}. Message: {e}. "
-            f"Response: {response.text}."
+            f"Request failed with status:{response.status_code}. " f"Response: {response.text}."
         )
         logger.warning(error_message)
         errors.append(error_message)
@@ -181,10 +178,8 @@ def get_sample_xml(sample_set):
         return files
 
     xml = get_sample_xml(sample_set)
-    try:
-        response = post_webin(config, xml)
-        response.raise_for_status()
-    except requests.exceptions.RequestException:
+    response = post_webin(config, xml)
+    if not response.ok:
         error_message = (
             f"Request failed with status:{response.status_code}. "
             f"Request: {response.request}, Response: {response.text}"
@@ -380,14 +375,14 @@ def check_ena(config: ENAConfig, erz_accession: str, segment_order: list[str]) -
     errors = []
     warnings = []
     assembly_results = {"segment_order": segment_order}
-    try:
-        response = requests.get(
-            url,
-            auth=HTTPBasicAuth(config.ena_submission_username, config.ena_submission_password),
-            timeout=10,  # wait a full 10 seconds for a response incase slow
-        )
-        response.raise_for_status()
-    except requests.exceptions.RequestException:
+
+    response = requests.get(
+        url,
+        auth=HTTPBasicAuth(config.ena_submission_username, config.ena_submission_password),
+        timeout=10,  # wait a full 10 seconds for a response incase slow
+    )
+    response.raise_for_status()
+    if not response.ok:
         error_message = (
             f"ENA check failed with status:{response.status_code}. "
             f"Request: {response.request}, Response: {response.text}"

ena-submission/scripts/submission_db_helper.py

@@ -26,7 +26,7 @@ def db_init(
 
     return SimpleConnectionPool(
         minconn=1,
-        maxconn=4,  # max 7*4 connections to db allowed
+        maxconn=2,  # max 7*2 connections to db allowed
         dbname="loculus",
         user=db_username,
         host=db_host,

ena-submission/scripts/test_ena_submission.py

@@ -93,6 +93,7 @@ def mock_requests_post(status_code, text):
     mock_response = mock.Mock()
     mock_response.status_code = status_code
     mock_response.text = text
+    mock_response.ok = mock_response.status_code < 400
     return mock_response
 
 
@@ -122,7 +123,6 @@ def test_create_project_xml_failure(self, mock_post):
     def test_create_project_server_failure(self, mock_post):
         # Testing project creation failure
         mock_post.return_value = mock_requests_post(500, "Internal Server Error")
-        mock_post.return_value.raise_for_status.side_effect = exceptions.RequestException()
         project_set = default_project_type()
         response = create_ena_project(test_ena_config, project_set)
         error_message_part = "Request failed with status:500"

ena-submission/scripts/trigger_submission_to_ena.py

@@ -77,7 +77,14 @@ def upload_sequences(db_config: SimpleConnectionPool, sequences_to_upload: dict[
     required=False,
     type=click.Path(),
 )
-def trigger_submission_to_ena(log_level, config_file, input_file=None):
+@click.option(
+    "--min-between-github-requests",
+    default=2,
+    type=int,
+)
+def trigger_submission_to_ena(
+    log_level, config_file, input_file=None, min_between_github_requests=2
+):
     logger.setLevel(log_level)
     logging.getLogger("requests").setLevel(logging.INFO)
 
@@ -109,7 +116,7 @@ def trigger_submission_to_ena(log_level, config_file, input_file=None):
             error_msg = f"Failed to retrieve file: {response.status_code}"
             logger.error(error_msg)
         upload_sequences(db_config, sequences_to_upload)
-        time.sleep(60)  # Sleep for 1min to not overwhelm github
+        time.sleep(min_between_github_requests * 60)  # Sleep for x min to not overwhelm github
 
 
 if __name__ == "__main__":

ena-submission/scripts/upload_external_metadata_to_loculus.py

@@ -208,7 +208,12 @@ def upload_handle_errors(
     required=True,
     type=click.Path(exists=True),
 )
-def upload_external_metadata(log_level, config_file):
+@click.option(
+    "--time-between-iterations",
+    default=10,
+    type=int,
+)
+def upload_external_metadata(log_level, config_file, time_between_iterations=10):
     logger.setLevel(log_level)
     logging.getLogger("requests").setLevel(logging.INFO)
 
@@ -231,7 +236,7 @@ def upload_external_metadata(log_level, config_file):
             config,
             slack_config,
         )
-        time.sleep(2)
+        time.sleep(time_between_iterations)
 
 
 if __name__ == "__main__":