feat(api): add support for remote project import from AWS S3 (#2357)

abhishekmsingh · web-flow · commit 892281e35e3d · 2022-11-02T09:29:06.000+01:00
diff --git a/docs/gl_objects/projects.rst b/docs/gl_objects/projects.rst
@@ -352,6 +352,20 @@ Import the project using file stored on a remote URL::
         override_params={'visibility': 'private'},
     )
 
+Import the project using file stored on AWS S3::
+
+    output = gl.projects.remote_import_s3(
+        path="my_new_remote_project",
+        region="aws-region",
+        bucket_name="aws-bucket-name",
+        file_key="aws-file-key",
+        access_key_id="aws-access-key-id",
+        secret_access_key="secret-access-key",
+        name="My New Remote Project", 
+        namespace="my-group",
+        override_params={'visibility': 'private'},
+    )
+
 Project custom attributes
 =========================
 
diff --git a/gitlab/v4/objects/projects.py b/gitlab/v4/objects/projects.py
@@ -807,6 +807,7 @@ class ProjectManager(CRUDMixin, RESTManager):
     def get(self, id: Union[str, int], lazy: bool = False, **kwargs: Any) -> Project:
         return cast(Project, super().get(id=id, lazy=lazy, **kwargs))
 
+    @exc.on_http_error(exc.GitlabImportError)
     def import_project(
         self,
         file: str,
@@ -833,7 +834,7 @@ def import_project(
 
         Raises:
             GitlabAuthenticationError: If authentication is not correct
-            GitlabListError: If the server failed to perform the request
+            GitlabImportError: If the server failed to perform the request
 
         Returns:
             A representation of the import status.
@@ -851,6 +852,7 @@ def import_project(
             "/projects/import", post_data=data, files=files, **kwargs
         )
 
+    @exc.on_http_error(exc.GitlabImportError)
     def remote_import(
         self,
         url: str,
@@ -877,7 +879,7 @@ def remote_import(
 
         Raises:
             GitlabAuthenticationError: If authentication is not correct
-            GitlabListError: If the server failed to perform the request
+            GitlabImportError: If the server failed to perform the request
 
         Returns:
             A representation of the import status.
@@ -894,6 +896,66 @@ def remote_import(
             "/projects/remote-import", post_data=data, **kwargs
         )
 
+    @exc.on_http_error(exc.GitlabImportError)
+    def remote_import_s3(
+        self,
+        path: str,
+        region: str,
+        bucket_name: str,
+        file_key: str,
+        access_key_id: str,
+        secret_access_key: str,
+        name: Optional[str] = None,
+        namespace: Optional[str] = None,
+        overwrite: bool = False,
+        override_params: Optional[Dict[str, Any]] = None,
+        **kwargs: Any,
+    ) -> Union[Dict[str, Any], requests.Response]:
+        """Import a project from an archive file stored on AWS S3.
+
+        Args:
+            region: AWS S3 region name where the file is stored
+            bucket_name: AWS S3 bucket name where the file is stored
+            file_key: AWS S3 file key to identify the file.
+            access_key_id: AWS S3 access key ID.
+            secret_access_key: AWS S3 secret access key.
+            path: Name and path for the new project
+            name: The name of the project to import. If not provided,
+                defaults to the path of the project.
+            namespace: The ID or path of the namespace that the project
+                will be imported to
+            overwrite: If True overwrite an existing project with the
+                same path
+            override_params: Set the specific settings for the project
+            **kwargs: Extra options to send to the server (e.g. sudo)
+
+        Raises:
+            GitlabAuthenticationError: If authentication is not correct
+            GitlabImportError: If the server failed to perform the request
+
+        Returns:
+            A representation of the import status.
+        """
+        data = {
+            "region": region,
+            "bucket_name": bucket_name,
+            "file_key": file_key,
+            "access_key_id": access_key_id,
+            "secret_access_key": secret_access_key,
+            "path": path,
+            "overwrite": str(overwrite),
+        }
+        if override_params:
+            for k, v in override_params.items():
+                data[f"override_params[{k}]"] = v
+        if name is not None:
+            data["name"] = name
+        if namespace:
+            data["namespace"] = namespace
+        return self.gitlab.http_post(
+            "/projects/remote-import-s3", post_data=data, **kwargs
+        )
+
     def import_bitbucket_server(
         self,
         bitbucket_server_url: str,
diff --git a/tests/functional/api/test_import_export.py b/tests/functional/api/test_import_export.py
@@ -69,7 +69,7 @@ def test_project_import_export(gl, project, temp_dir):
 
 
 def test_project_remote_import(gl):
-    with pytest.raises(gitlab.exceptions.GitlabHttpError) as err_info:
+    with pytest.raises(gitlab.exceptions.GitlabImportError) as err_info:
         gl.projects.remote_import(
             "ftp://whatever.com/url", "remote-project", "remote-project", "root"
         )
@@ -78,3 +78,23 @@ def test_project_remote_import(gl):
         "File url is blocked: Only allowed schemes are https"
         in err_info.value.error_message
     )
+
+
+def test_project_remote_import_s3(gl):
+    gl.features.set("import_project_from_remote_file_s3", True)
+    with pytest.raises(gitlab.exceptions.GitlabImportError) as err_info:
+        gl.projects.remote_import_s3(
+            "remote-project",
+            "aws-region",
+            "aws-bucket-name",
+            "aws-file-key",
+            "aws-access-key-id",
+            "secret-access-key",
+            "remote-project",
+            "root",
+        )
+    assert err_info.value.response_code == 400
+    assert (
+        "Failed to open 'aws-file-key' in 'aws-bucket-name'"
+        in err_info.value.error_message
+    )
diff --git a/tests/unit/objects/test_project_import_export.py b/tests/unit/objects/test_project_import_export.py
@@ -53,6 +53,30 @@ def resp_remote_import():
         yield rsps
 
 
+@pytest.fixture
+def resp_remote_import_s3():
+    content = {
+        "id": 1,
+        "description": None,
+        "name": "remote-project-s3",
+        "name_with_namespace": "Administrator / remote-project-s3",
+        "path": "remote-project-s3",
+        "path_with_namespace": "root/remote-project-s3",
+        "created_at": "2018-02-13T09:05:58.023Z",
+        "import_status": "scheduled",
+    }
+
+    with responses.RequestsMock() as rsps:
+        rsps.add(
+            method=responses.POST,
+            url="http://localhost/api/v4/projects/remote-import-s3",
+            json=content,
+            content_type="application/json",
+            status=200,
+        )
+        yield rsps
+
+
 @pytest.fixture
 def resp_import_status():
     content = {
@@ -125,7 +149,24 @@ def test_import_project(gl, resp_import_project):
 
 def test_remote_import(gl, resp_remote_import):
     project_import = gl.projects.remote_import(
-        "https://whatever.com/url", "remote-project", "remote-project", "root"
+        "https://whatever.com/url/file.tar.gz",
+        "remote-project",
+        "remote-project",
+        "root",
+    )
+    assert project_import["import_status"] == "scheduled"
+
+
+def test_remote_import_s3(gl, resp_remote_import_s3):
+    project_import = gl.projects.remote_import_s3(
+        "remote-project",
+        "aws-region",
+        "aws-bucket-name",
+        "aws-file-key",
+        "aws-access-key-id",
+        "secret-access-key",
+        "remote-project",
+        "root",
     )
     assert project_import["import_status"] == "scheduled"
 
