From 81e67d1f09123ee210576a86627c7757d08776b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EB=AA=85=EC=A7=84?= Date: Fri, 23 Feb 2024 15:02:06 +0900 Subject: [PATCH] Add description --- main.py | 2 +- src/core.py | 80 ++++++++++++++++++++++++++++++++++++++++------------- src/s3.py | 5 ++-- 3 files changed, 65 insertions(+), 22 deletions(-) diff --git a/main.py b/main.py index eb3a5a5..609f9a2 100644 --- a/main.py +++ b/main.py @@ -137,7 +137,7 @@ def config(): def run(user_input_path: Path): """ Run obs3dian main command, get image local file paths from md file. - After extracting image paths upload images to s3 and replace all file links in .md to S3 links. + After extracting image paths, it uploads images to s3 and replaces all file links in .md to S3 links. Output.md will be wrriten under output folder and it's image links would have been replaced to S3 links Args: diff --git a/src/core.py b/src/core.py index 1093dc4..04dbac4 100644 --- a/src/core.py +++ b/src/core.py @@ -7,42 +7,84 @@ from typing import Generator, Callable, List -def create_s3_key(markdown_file_name: str, path: Path) -> str: - return f"{markdown_file_name}/{path.name}" +def create_s3_key(markdown_file_name: str, image_name: str) -> str: + """ + Create key by makrdown file name and image name key is used in S3 + + Args: + markdown_file_name (str): mark down file name + image_name (str): image file name in markdown + + Returns: + str: S3 key + """ + return f"{markdown_file_name}/{image_name}" def put_images_in_md( - s3: S3, markdown_file_name: str, path_generator: Generator[Path, None, None] + s3: S3, markdown_file_name: str, image_path_generator: Generator[Path, None, None] ) -> List[Path]: + """ + Generate local images path by using generator and put images to S3. + Return sccessfully put image paths + + Args: + s3 (S3): instance to control S3 + markdown_file_name (str): makrdown file name + image_path_generator (Generator[Path, None, None]): yield image paths to upload + + Returns: + List[Path]: sccessfully put image paths + """ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: futures = [] - for path in path_generator: # s3 key - s3_key = create_s3_key(markdown_file_name, path) - futures.append(executor.submit(s3.put_image, path, s3_key)) + for path in image_path_generator: + s3_key = create_s3_key(markdown_file_name, path.name) # create Key + futures.append( + executor.submit(s3.put_image, path, s3_key) + ) # run upload by multithread - links_map = [] + put_image_paths: List[Path] = [] # put success image path list for future in concurrent.futures.as_completed(futures): try: - links_map.append(future.result()) + put_image_paths.append(future.result()) except Exception as e: - print("Erorr Occured: ", str(e)) + continue - return links_map + return put_image_paths def create_obs3dian_runner(s3: S3, output_folder_path: Path) -> Callable: - def run_obs3dian(markdown_file_path: Path) -> None: - path_generator: Generator = generate_local_paths(markdown_file_path) + """ + Create runner fucntion object + S3 controller and ouput_folder_path would not change before config + + Args: + s3 (S3): S3 controller + output_folder_path (Path): output folder path + + Returns: + Callable: runner + """ + + def run(markdown_file_path: Path) -> None: + """ + Run obs3dian command extract image paths and replace them by S3 URLs. + + Args: + markdown_file_path (Path): mark down file path + """ + image_path_generator: Generator = generate_local_paths(markdown_file_path) markdown_file_name = markdown_file_path.stem - uploaded_image_paths = put_images_in_md(s3, markdown_file_name, path_generator) + put_image_paths = put_images_in_md(s3, markdown_file_name, image_path_generator) - link_replace_map = [ - (path.name, s3.get_s3_url(create_s3_key(markdown_file_name, path))) - for path in uploaded_image_paths - ] + link_replace_pairs = [ + (path.name, s3.get_s3_url(create_s3_key(markdown_file_name, path.name))) + for path in put_image_paths + ] # get (local image path, S3 URL) to convert link - write_md_file(markdown_file_path, output_folder_path, link_replace_map) + write_md_file(markdown_file_path, output_folder_path, link_replace_pairs) return - return run_obs3dian + return run diff --git a/src/s3.py b/src/s3.py index 5788833..3b23d12 100644 --- a/src/s3.py +++ b/src/s3.py @@ -82,7 +82,7 @@ def get_s3_url(self, key: str) -> str: ) return s3_url - def put_image(self, file_path: Path, key: str) -> Path | None: + def put_image(self, file_path: Path, key: str) -> Path: try: self.s3.put_object( Bucket=self.bucket_name, @@ -92,4 +92,5 @@ def put_image(self, file_path: Path, key: str) -> Path | None: return file_path except ClientError as e: - print("Error Occured in uploading...\n", e) + print(f"Error Occured in uploading {file_path}") + raise e