feat!: remove s3 upload feature (#1095)

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Removed
+
+- BREAKING: drop the s3 upload feature (#1095)
+
 ## [0.17.0] 2022-06-07
 
 ### Added

backend/requirements.txt

@@ -18,7 +18,6 @@ psycopg2-binary==2.9.3
 requests>=2.20.0
 uwsgi==2.0.20
 zxcvbn==4.4.28
-boto3==1.24.0
 grpcio==1.46.3
 grpcio-tools==1.46.3
 minio==7.1.8

backend/substrapp/compute_tasks/command.py

@@ -10,8 +10,6 @@
 from substrapp.compute_tasks.context import Context
 from substrapp.compute_tasks.directories import SANDBOX_DIR
 from substrapp.compute_tasks.directories import TaskDirName
-from substrapp.compute_tasks.transfer_bucket import TAG_VALUE_FOR_TRANSFER_BUCKET
-from substrapp.compute_tasks.transfer_bucket import TRANSFER_BUCKET_TESTTUPLE_TAG
 from substrapp.models.image_entrypoint import ImageEntrypoint
 
 logger = structlog.get_logger(__name__)
@@ -53,10 +51,9 @@ def get_exec_command(ctx: Context, algo: Algo, is_testtuple_eval: bool) -> List[
     if command[0].startswith("python"):
         command.insert(1, "-u")  # unbuffered. Allows streaming the logs in real-time.
 
-    env = _get_env(ctx, is_testtuple_eval)
     args = _get_args(ctx, algo.key, is_testtuple_eval)
 
-    return env + command + args
+    return command + args
 
 
 class TaskResource(dict):
@@ -199,17 +196,3 @@ def _get_args(ctx: Context, algo_key: str, is_testtuple_eval: bool) -> List[str]
     logger.debug("Generated task command", command=command)
 
     return command
-
-
-def _get_env(ctx: Context, is_testtuple_eval: bool) -> List[str]:
-    """This return environment variables for the task"""
-
-    env = []
-
-    # Transfer bucket
-    tag = ctx.task.get("tag")
-    if ctx.task_category == computetask_pb2.TASK_TEST and not is_testtuple_eval:
-        if tag and TAG_VALUE_FOR_TRANSFER_BUCKET in tag:
-            env.append(f"{TRANSFER_BUCKET_TESTTUPLE_TAG}={TAG_VALUE_FOR_TRANSFER_BUCKET}")
-
-    return env

backend/substrapp/tasks/tasks_compute_task.py

@@ -51,8 +51,6 @@
 from substrapp.compute_tasks.lock import MAX_TASK_DURATION
 from substrapp.compute_tasks.lock import acquire_compute_plan_lock
 from substrapp.compute_tasks.save_models import save_models
-from substrapp.compute_tasks.transfer_bucket import TAG_VALUE_FOR_TRANSFER_BUCKET
-from substrapp.compute_tasks.transfer_bucket import transfer_to_bucket
 from substrapp.lock_local import lock_resource
 from substrapp.orchestrator import get_orchestrator_client
 from substrapp.utils import list_dir
@@ -226,8 +224,6 @@ def _run(self, channel_name: str, task, compute_plan_key):  # noqa: C901
                 result["result"] = {"performances": {}}
                 for metric in ctx.metrics:
                     result["result"]["performances"][metric.key] = _get_perf(dirs, metric.key)
-
-                _transfer_model_to_bucket(ctx)
             else:
                 logger.info("Saving models and local folder")
                 save_models(ctx)
@@ -271,13 +267,6 @@ def _prepare_chainkeys(compute_plan_dir: str, compute_plan_tag: str):
     prepare_chainkeys_dir(chainkeys_dir, compute_plan_tag)  # does nothing if chainkeys already populated
 
 
-def _transfer_model_to_bucket(ctx: Context) -> None:
-    """Export model to S3 bucket if the task has appropriate tag"""
-    if ctx.task["tag"] and TAG_VALUE_FOR_TRANSFER_BUCKET in ctx.task["tag"]:
-        logger.info("Task eligible to bucket export")
-        transfer_to_bucket(ctx)
-
-
 def _store_failure(exc: Exception, compute_task_key: str) -> Optional[models.ComputeTaskFailureReport]:
     """If the provided exception is an `ExecutionError`, store its logs in the Django storage and in the database.
     Otherwise, do nothing.