From 6240ebad356afb415069adfbabbebe21fd8b908b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=A0=95=EB=AA=85=EC=A7=84?= Date: Sun, 25 Feb 2024 23:08:30 +0900 Subject: [PATCH] Remove create_key function --- obs3dian/src/core.py | 12 +++--- obs3dian/src/s3.py | 94 ++++++++++++++++++++++---------------------- 2 files changed, 52 insertions(+), 54 deletions(-) diff --git a/obs3dian/src/core.py b/obs3dian/src/core.py index c846409..efda746 100644 --- a/obs3dian/src/core.py +++ b/obs3dian/src/core.py @@ -8,7 +8,7 @@ def put_images_in_md( - s3: S3, markdown_file_name: str, image_path_generator: Generator[Path, None, None] + s3: S3, markdown_path: Path, image_path_generator: Generator[Path, None, None] ) -> List[Path]: """ Generate local images path by using generator and put images to S3. @@ -24,10 +24,9 @@ def put_images_in_md( """ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor: futures = [] - for path in image_path_generator: - s3_key = s3.create_s3_key(markdown_file_name, path.name) # create Key + for image_path in image_path_generator: futures.append( - executor.submit(s3.put_image, path, s3_key) + executor.submit(s3.put_image, markdown_path, image_path) ) # run upload by multithread put_image_paths: List[Path] = [] # put success image path list @@ -66,15 +65,14 @@ def run(markdown_file_path: Path) -> None: image_path_generator: Generator = generate_local_image_paths( image_folder_path, markdown_file_path ) - markdown_file_name = markdown_file_path.stem - put_image_paths = put_images_in_md(s3, markdown_file_name, image_path_generator) + put_image_paths = put_images_in_md(s3, markdown_file_path, image_path_generator) # (image name, S3 URL) to convert link link_replace_pairs: List[Tuple[str, str]] = [] for image_path in put_image_paths: image_name = image_path.name - s3_url = s3.get_image_url(markdown_file_name, image_name) + s3_url = s3.get_image_url(markdown_file_path, image_path) link_replace_pairs.append( (image_name, s3_url) ) # image name in .md would convert to S3 url diff --git a/obs3dian/src/s3.py b/obs3dian/src/s3.py index 1285651..12e927e 100644 --- a/obs3dian/src/s3.py +++ b/obs3dian/src/s3.py @@ -17,19 +17,6 @@ def __init__(self, profile_name: str, bucket_name: str) -> None: print("Can't Connect S3") raise e - def create_s3_key(self, markdown_file_name: str, image_name: str) -> str: - """ - Create key by makrdown file name and image name key is used in S3 - - Args: - markdown_file_name (str): mark down file name - image_name (str): image file name in markdown - - Returns: - str: S3 key - """ - return f"{markdown_file_name}/{image_name}" - def _check_bucket_exist(self) -> bool: try: self.s3.head_bucket(Bucket=self.bucket_name) @@ -37,13 +24,37 @@ def _check_bucket_exist(self) -> bool: except ClientError as e: error_code = e.response["Error"]["Code"] - assert error_code == "404", "Forbidden or Badrequest on S3" + if error_code != "404": # if no permission or bad request raise error + raise e return False + def put_public_access_policy(self) -> None: + policy = { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "id-1", + "Action": ["s3:GetObject"], + "Effect": "Allow", + "Resource": f"arn:aws:s3:::{self.bucket_name}/*", + "Principal": "*", + } + ], + } + try: + self.s3.put_bucket_policy( + Bucket=self.bucket_name, Policy=json.dumps(policy) + ) + + except Exception as e: + print("Can't allow public acess to bucket") + raise e + def create_bucket(self) -> bool: - if self._check_bucket_exist(): - return False try: + if self._check_bucket_exist(): # If bucket already exists + return False + bucket_name = self.bucket_name self.s3.create_bucket( CreateBucketConfiguration={"LocationConstraint": "ap-northeast-2"}, @@ -63,48 +74,37 @@ def create_bucket(self) -> bool: return True except ClientError as e: - print("Error occured in create bucket ", e) + print("Error occured in create bucket") raise e - def put_public_access_policy(self) -> None: - policy = { - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "id-1", - "Action": ["s3:GetObject"], - "Effect": "Allow", - "Resource": f"arn:aws:s3:::{self.bucket_name}/*", - "Principal": "*", - } - ], - } - try: - self.s3.put_bucket_policy( - Bucket=self.bucket_name, Policy=json.dumps(policy) - ) + # def create_s3_key(self, markdown_file_name: str, image_name: str) -> str: + # """ + # Create key by makrdown file name and image name. - except Exception as e: - print("Can't allow public acess to bucket check permission") - raise e + # Args: + # markdown_file_name (str): mark down file name + # image_name (str): image file name in markdown + + # Returns: + # str: S3 key + # """ + # return f"{markdown_file_name}/{image_name}" - def get_image_url(self, markdown_file_name: str, image_name: str) -> str: + def get_image_url(self, markdown_path: Path, image_path: Path) -> str: region = self.session.region_name - key = self.create_s3_key(markdown_file_name, image_name) - s3_url = ( - f"https://{self.bucket_name}.s3.{region}.amazonaws.com/{parse.quote(key)}" - ) + key = markdown_path / image_path + s3_url = f"https://{self.bucket_name}.s3.{region}.amazonaws.com/{parse.quote(str(key))}" return s3_url - def put_image(self, file_path: Path, key: str) -> Path: + def put_image(self, markdown_path: Path, image_path: Path) -> Path: try: self.s3.put_object( Bucket=self.bucket_name, - Body=file_path.open("rb"), - Key=key, + Body=image_path.open("rb"), + Key=str(markdown_path / image_path), ) - return file_path + return image_path except ClientError as e: - print(f"Error Occured in uploading {file_path}") + print(f"Error Occured in uploading {image_path}") raise e