Skip to content

added destination to case.results.download() functionality #53

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 28, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 4 additions & 7 deletions examples/case_results.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,15 +27,12 @@
print(case.results.minmax_state.raw)

# download all result files of the case:
case.results.download_manager(all=True)
downloaded_files = case.results.download(all=True, destination=case.name)
print(downloaded_files)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should we print, or refactor to use log [Potentially allow centralized control over printing by setting logging levels]

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is part of examples. I think it is better to use print in examples as these will be run and modify by users so they will not be familiar with log.


# download specific result files of the case:
case.results.download_manager(bet_forces=True, actuator_disk_output=True)

# alternative way of downloading using dedicated functions:
case.results.download_surface()
case.results.download_volumetric()

# download specific result files of the case:
case.results.download(bet_forces=True, actuator_disk_output=True)

try:
case.results.total_forces.plot()
Expand Down
62 changes: 49 additions & 13 deletions flow360/cloud/s3_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
)

from ..environment import Env
from ..exceptions import CloudFileError
from ..exceptions import ValueError as FlValueError
from ..log import log
from .http_util import http

Expand Down Expand Up @@ -52,17 +52,47 @@ def __call__(self, bytes_chunk_transferred):
)


def create_base_folder(path: str, target_name: str, to_file: str = ".", keep_folder: bool = True):
def create_base_folder(
path: str, target_name: str, to_file: str = ".", to_folder: str = ".", keep_folder: bool = True
):
"""
:param path: source id
:param target_name: path to file on cloud, same value as key for s3 client download.
:param to_file: could be either folder or file name.
:param keep_folder: If true, the downloaded file will
be put in the same folder as the file on cloud. Only work
when file_name is a folder name.
:return:
Create a base folder and return the target file path for downloading cloud data.

Parameters
----------
path : str
Source ID or the path to the source file on the cloud.

target_name : str
The file path on the cloud, same value as the key for S3 client download.

to_file : str, optional
The destination folder or file path where the downloaded file will be saved. If None, the current directory
will be used.

to_folder : str, optional
The folder name to save the downloaded file. If provided, the downloaded file will be saved inside this folder.
If None, the value of `to_file` will be considered as a folder or file path.

keep_folder : bool, optional
If True, the downloaded file will be put in the same folder as the file on the cloud (only works when
`target_name` is a folder name).

Returns
-------
str
The target file path for downloading cloud data.

"""
if os.path.isdir(to_file):

if to_folder != ".":
to_file = (
os.path.join(to_folder, target_name)
if keep_folder
else os.path.join(to_folder, os.path.basename(target_name))
)

elif os.path.isdir(to_file):
to_file = (
os.path.join(to_file, path, target_name)
if keep_folder
Expand Down Expand Up @@ -226,7 +256,8 @@ def download_file(
self,
resource_id: str,
remote_file_name: str,
to_file: str,
to_file: str = ".",
to_folder: str = ".",
keep_folder: bool = True,
overwrite: bool = True,
progress_callback=None,
Expand All @@ -243,10 +274,14 @@ def download_file(
:param progress_callback: provide custom callback for progress
:return:
"""
to_file = create_base_folder(resource_id, remote_file_name, to_file, keep_folder)
if to_file != "." and to_folder != ".":
raise FlValueError("Only one of 'to_file' or 'to_folder' should be provided, not both.")

to_file = create_base_folder(resource_id, remote_file_name, to_file, to_folder, keep_folder)
if os.path.exists(to_file) and not overwrite:
log.info(f"Skipping {remote_file_name}, file exists.")
return
return to_file

token = self._get_s3_sts_token(resource_id, remote_file_name)
client = token.get_client()
try:
Expand Down Expand Up @@ -283,6 +318,7 @@ def _call_back(bytes_in_chunk):
Callback=_call_back,
)
log.info(f"Saved to {to_file}")
return to_file

def _get_s3_sts_token(self, resource_id: str, file_name: str) -> _S3STSToken:
session_key = f"{resource_id}:{self.value}:{file_name}"
Expand Down
136 changes: 81 additions & 55 deletions flow360/component/case.py
Original file line number Diff line number Diff line change
Expand Up @@ -447,18 +447,6 @@ def results(self) -> CaseResults:
"""
return self._results

def download_log(self, log_file, to_file=".", keep_folder: bool = True):
"""
Download log
:param log_file:
:param to_file: file name on local disk, could be either folder or file name.
:param keep_folder: If true, the downloaded file will be put in the same folder as the file on cloud. Only work
when file_name is a folder name.
:return:
"""

self.download_file(f"logs/{log_file.value}", to_file, keep_folder)

def is_steady(self):
"""
returns True when case is steady state
Expand Down Expand Up @@ -770,30 +758,52 @@ def plot(self):
"""
return self._plotter

def download_file(self, downloadable: CaseDownloadable, overwrite: bool = True, **kwargs):
"""
download specific file by filename
:param downloadable: filename to download
:param overwrite: when True, overwrites existing file, otherwise skip
# pylint: disable=protected-access
def _download_file(
self,
downloadable: CaseDownloadable,
to_file=".",
to_folder=".",
overwrite: bool = True,
**kwargs,
):
"""
return self._case.download_file(
f"results/{downloadable.value}", overwrite=overwrite, **kwargs
)
Download a specific file associated with the case.

def download_volumetric(self):
"""
download volumetric results data
"""
self.download_file(CaseDownloadable.VOLUME)
Parameters
----------
downloadable : flow360.CaseDownloadable
The type of file to be downloaded (e.g., surface, volume, forces, etc.).

def download_surface(self):
"""
download surface results data
to_file : str, optional
File path to save the downloaded file. If None, the file will be saved in the current directory.
If provided without an extension, the extension will be automatically added based on the file type.

to_folder : str, optional
Folder name to save the downloaded file. If None, the file will be saved in the current directory.

overwrite : bool, optional
If True, overwrite existing files with the same name in the destination.

**kwargs : dict, optional
Additional arguments to be passed to the download process.

Returns
-------
str
File path of the downloaded file.
"""
self.download_file(CaseDownloadable.SURFACE)

return self._case._download_file(
f"results/{downloadable.value}",
to_file=to_file,
to_folder=to_folder,
overwrite=overwrite,
**kwargs,
)

# pylint: disable=redefined-builtin,too-many-locals,too-many-arguments
def download_manager(
def download(
self,
surface: bool = False,
volume: bool = False,
Expand All @@ -807,42 +817,44 @@ def download_manager(
actuator_disk_output: bool = False,
all: bool = False,
overwrite: bool = False,
destination: str = ".",
):
"""download manager for downloading many files at once
"""
Download result files associated with the case.

Parameters
----------
surface : bool, optional
_description_, by default False
Download surface result file if True.
volume : bool, optional
_description_, by default False
Download volume result file if True.
nonlinear_residuals : bool, optional
_description_, by default False
Download nonlinear residuals file if True.
linear_residuals : bool, optional
_description_, by default False
Download linear residuals file if True.
cfl : bool, optional
_description_, by default False
Download CFL file if True.
minmax_state : bool, optional
_description_, by default False
Download minmax state file if True.
surface_forces : bool, optional
_description_, by default False
Download surface forces file if True.
total_forces : bool, optional
_description_, by default False
Download total forces file if True.
bet_forces : bool, optional
_description_, by default False
Download BET (Blade Element Theory) forces file if True.
actuator_disk_output : bool, optional
_description_, by default False
Download actuator disk output file if True.
all : bool, optional
_description_, by default False
Download all result files if True (ignores other parameters).
overwrite : bool, optional
_description_, by default False
If True, overwrite existing files with the same name in the destination.
destination : str, optional
Location to save downloaded files. If None, files will be saved in the current directory under ID folder.

Raises
------
e
_description_
e
_description_
Returns
-------
List of str
File paths of the downloaded files.
"""

download_map = [
Expand All @@ -855,15 +867,22 @@ def download_manager(
(surface_forces, CaseDownloadable.SURFACE_FORCES),
(total_forces, CaseDownloadable.TOTAL_FORCES),
]

downloaded_files = []
for do_download, filename in download_map:
if do_download or all:
self.download_file(filename, overwrite=overwrite)
downloaded_files.append(
self._download_file(filename, to_folder=destination, overwrite=overwrite)
)

if bet_forces or all:
try:
self.download_file(
CaseDownloadable.BET_FORCES, overwrite=overwrite, log_error=False
downloaded_files.append(
self._download_file(
CaseDownloadable.BET_FORCES,
to_folder=destination,
overwrite=overwrite,
log_error=False,
)
)
except CloudFileNotFoundError as err:
if not self._case.has_bet_disks():
Expand All @@ -877,8 +896,13 @@ def download_manager(

if actuator_disk_output or all:
try:
self.download_file(
CaseDownloadable.ACTUATOR_DISK_OUTPUT, overwrite=overwrite, log_error=False
downloaded_files.append(
self._download_file(
CaseDownloadable.ACTUATOR_DISK_OUTPUT,
to_folder=destination,
overwrite=overwrite,
log_error=False,
)
)
except CloudFileNotFoundError as err:
if not self._case.has_actuator_disks():
Expand All @@ -893,6 +917,8 @@ def download_manager(
)
raise err

return downloaded_files


class CaseList(Flow360ResourceListBase):
"""
Expand Down
Loading