add auth

.github/workflows/codestyle.yml

@@ -31,7 +31,7 @@ jobs:
         with:
           python-version: '3.9'
           cache: 'poetry'
-      - name: Install pylint
-        run: poetry add pylint -D
+      - name: Install dependencies
+        run: poetry install
       - name: Run pylint
-        run: pylint flow360 --rcfile .pylintrc
+        run: poetry run pylint flow360 --rcfile .pylintrc

.github/workflows/test.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ${{ matrix.platform }}
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.7', '3.8', '3.9', '3.10']
         platform: [ubuntu-latest, macos-latest, windows-latest]
     steps:
     - uses: actions/checkout@v3

.pylintrc

@@ -51,7 +51,7 @@ ignore-paths=tests
 # Files or directories matching the regex patterns are skipped. The regex
 # matches against base names, not paths. The default value ignores Emacs file
 # locks
-ignore-patterns=test_.*?py
+ignore-patterns=test_.*?py, _test_.*?py
 
 # List of module names for which member attributes should not be checked
 # (useful for modules/projects where namespaces are manipulated during runtime
@@ -426,7 +426,7 @@ min-similarity-lines=4
 
 # List of regular expressions of class ancestor names to ignore when counting
 # public methods (see R0903)
-exclude-too-few-public-methods=True
+exclude-too-few-public-methods=
 
 # List of qualified class names to ignore when counting class parents (see
 # R0901)

flow360/cloud/http.py → flow360/cloud/http_util.py

@@ -1,5 +1,6 @@
 """
-http utils.
+http utils. Example:
+http.get(path)
 """
 import requests
 
@@ -35,7 +36,7 @@ def get(self, path: str, json=None):
         :param json:
         :return:
         """
-        return self.session.get(Env.current.get_real_url(path), auth=api_key_auth, json=json)
+        return self.session.get(url=Env.current.get_real_url(path), auth=api_key_auth, json=json)
 
     def post(self, path: str, json):
         """
@@ -64,4 +65,4 @@ def delete(self, path: str):
         return self.session.delete(Env.current.get_real_url(path), auth=api_key_auth)
 
 
-http = Http(requests.Session)
+http = Http(requests.Session())

flow360/cloud/s3_utils.py

@@ -0,0 +1,209 @@
+"""
+s3 util file for material uploading and downloading.
+"""
+import os
+import urllib
+from datetime import datetime
+from enum import Enum
+
+import boto3
+from boto3.s3.transfer import TransferConfig
+from pydantic import BaseModel, Field
+from rich.progress import (
+    BarColumn,
+    DownloadColumn,
+    Progress,
+    TextColumn,
+    TimeRemainingColumn,
+    TransferSpeedColumn,
+)
+
+from flow360.cloud.http_util import http
+from flow360.environment import Env
+
+_s3_config = TransferConfig(
+    multipart_threshold=1024 * 25,
+    max_concurrency=50,
+    multipart_chunksize=1024 * 25,
+    use_threads=True,
+)
+
+
+class _S3Action(Enum):
+    """
+    Enum for s3 action
+    """
+
+    UPLOADING = "↑"
+    DOWNLOADING = "↓"
+
+
+def _get_progress(action: _S3Action):
+    col = (
+        TextColumn(f"[bold green]{_S3Action.DOWNLOADING.value}")
+        if action == _S3Action.DOWNLOADING
+        else TextColumn(f"[bold red]{_S3Action.UPLOADING.value}")
+    )
+    return Progress(
+        col,
+        TextColumn("[bold blue]{task.fields[filename]}"),
+        BarColumn(),
+        "[progress.percentage]{task.percentage:>3.1f}%",
+        "•",
+        DownloadColumn(),
+        "•",
+        TransferSpeedColumn(),
+        "•",
+        TimeRemainingColumn(),
+    )
+
+
+class _UserCredential(BaseModel):
+    access_key_id: str = Field(alias="accessKeyId")
+    expiration: datetime
+    secret_access_key: str = Field(alias="secretAccessKey")
+    session_token: str = Field(alias="sessionToken")
+
+
+class _S3STSToken(BaseModel):
+    cloud_path: str = Field(alias="cloudpath")
+    user_credential: _UserCredential = Field(alias="userCredentials")
+
+    def get_bucket(self):
+        """
+        Get bucket name.
+        :return:
+        """
+        path = urllib.parse.urlparse(self.cloud_path)
+        return path.netloc
+
+    def get_s3_key(self):
+        """
+        Get s3 key.
+        :return:
+        """
+        path = urllib.parse.urlparse(self.cloud_path)
+        return path.path[1:]
+
+    def get_client(self):
+        """
+        Get s3 client.
+        :return:
+        """
+        return boto3.client(
+            "s3",
+            region_name=Env.current.aws_region,
+            aws_access_key_id=self.user_credential.access_key_id,
+            aws_secret_access_key=self.user_credential.secret_access_key,
+            aws_session_token=self.user_credential.session_token,
+        )
+
+    def is_expired(self):
+        """
+        Check if the token is expired.
+        :return:
+        """
+        return (
+            self.user_credential.expiration
+            - datetime.now(tz=self.user_credential.expiration.tzinfo)
+        ).total_seconds() > 300
+
+
+class S3TransferType(Enum):
+    """
+    Enum for s3 transfer type
+    """
+
+    VOLUME_MESH = "VolumeMesh"
+    SURFACE_MESH = "SurfaceMesh"
+    CASE = "Case"
+    STUDIO = "Studio"
+
+    def _get_grant_url(self, resource_id, file_name: str) -> str:
+        """
+        Get the grant url for a file.
+        :param resource_id:
+        :param file_name:
+        :return:
+        """
+        if self is S3TransferType.VOLUME_MESH:
+            return f"volumemeshes/{resource_id}/file?filename={file_name}"
+        if self is S3TransferType.SURFACE_MESH:
+            return f"surfacemeshes/{resource_id}/file?filename={file_name}"
+        if self is S3TransferType.CASE:
+            return f"cases/{resource_id}/file?filename={file_name}"
+        if self is S3TransferType.STUDIO:
+            return f"volumemeshes/{resource_id}/file?filename={file_name}"
+
+        return None
+
+    def upload_file(self, resource_id: str, remote_file_name: str, file_name: str):
+        """
+        Upload a file to s3.
+        :param resource_id:
+        :param remote_file_name:
+        :param file_name:
+        :return:
+        """
+        if not os.path.exists(file_name):
+            print(f"mesh file {file_name} does not Exist!")
+            raise FileNotFoundError()
+        with _get_progress(_S3Action.UPLOADING) as progress:
+            task_id = progress.add_task(
+                "upload", filename=os.path.basename(file_name), total=os.path.getsize(file_name)
+            )
+
+            def _call_back(bytes_in_chunk):
+                progress.update(task_id, advance=bytes_in_chunk)
+
+            token = self._get_s3_sts_token(resource_id, remote_file_name)
+            token.get_client().upload_file(
+                Bucket=token.get_bucket(),
+                Filename=file_name,
+                Key=token.get_s3_key(),
+                Callback=_call_back,
+                Config=_s3_config,
+            )
+
+    def download_file(self, resource_id: str, remote_file_name: str, to_file: str):
+        """
+        Download a file from s3.
+        :param resource_id:
+        :param remote_file_name:
+        :param to_file:
+        :return:
+        """
+        token = self._get_s3_sts_token(resource_id, remote_file_name)
+        client = token.get_client()
+        meta_data = client.head_object(Bucket=token.get_bucket(), Key=token.get_s3_key())
+        with _get_progress(_S3Action.DOWNLOADING) as progress:
+            progress.start()
+            task_id = progress.add_task(
+                "download",
+                filename=os.path.basename(remote_file_name),
+                total=meta_data.get("ContentLength", 0),
+            )
+
+            def _call_back(bytes_in_chunk):
+                progress.update(task_id, advance=bytes_in_chunk)
+
+            os.makedirs(os.path.dirname(os.path.abspath(to_file)), exist_ok=True)
+
+            client.download_file(
+                Bucket=token.get_bucket(),
+                Filename=to_file,
+                Key=token.get_s3_key(),
+                Callback=_call_back,
+            )
+
+    def _get_s3_sts_token(self, resource_id: str, file_name: str) -> _S3STSToken:
+        session_key = f"{resource_id}:{self.value}:{file_name}"
+        if session_key not in _s3_sts_tokens or _s3_sts_tokens[session_key].is_expired():
+            path = self._get_grant_url(resource_id, file_name)
+            resp = http.get(path)
+            token = _S3STSToken.parse_obj(resp)
+            _s3_sts_tokens[session_key] = token
+        return _s3_sts_tokens[session_key]
+
+
+_s3_sts_tokens: [str, _S3STSToken] = {}

flow360/component/surface_mesh.py

@@ -0,0 +1,36 @@
+"""
+Surface mesh component
+"""
+from datetime import datetime
+
+from pydantic import BaseModel, Extra, Field
+
+from flow360.cloud.http_util import http
+
+
+class SurfaceMesh(BaseModel, extra=Extra.allow):
+    """
+    Surface mesh component
+    """
+
+    surface_mesh_id: str = Field(..., alias="id")
+    name: str
+    status: str
+    user_id: str = Field(alias="userId")
+    solver_version: str = Field(alias="solverVersion")
+    config: str
+    created_at: str = Field(alias="createdAt")
+    updated_at: datetime = Field(alias="updatedAt")
+    updated_by: str = Field(alias="updatedBy")
+
+    @classmethod
+    def from_cloud(cls, mesh_id: str):
+        """
+        Get surface mesh info from cloud
+        :param mesh_id:
+        :return:
+        """
+        resp = http.get(f"surfacemeshes/{mesh_id}")
+        if resp.status_code == 200:
+            return cls(**resp.json()["data"])
+        return None

flow360/environment.py

@@ -12,6 +12,7 @@ class EnvironmentConfig(BaseModel):
 
     name: str
     web_api_endpoint: str
+    aws_region: str
 
     def active(self):
         """
@@ -29,11 +30,23 @@ def get_real_url(self, path: str):
         return "/".join([self.web_api_endpoint, path])
 
 
-dev = EnvironmentConfig(name="dev", web_api_endpoint="https://dev-tidy3d-api.simulation.cloud")
+dev = EnvironmentConfig(
+    name="dev",
+    web_api_endpoint="https://flow360-api.dev-simulation.cloud",
+    aws_region="us-gov-west-1",
+)
 
-uat = EnvironmentConfig(name="uat", web_api_endpoint="https://uat-tidy3d-api.simulation.cloud")
+uat = EnvironmentConfig(
+    name="uat",
+    web_api_endpoint="https://uat-flow360-api.simulation.cloud",
+    aws_region="us-gov-west-1",
+)
 
-prod = EnvironmentConfig(name="prod", web_api_endpoint="https://prod-tidy3d-api.simulation.cloud")
+prod = EnvironmentConfig(
+    name="prod",
+    web_api_endpoint="https://prod-flow360-api.simulation.cloud",
+    aws_region="us-east-1",
+)
 
 
 class Environment: