Skip to content

Commit

Permalink
Remove create_key function
Browse files Browse the repository at this point in the history
  • Loading branch information
thinkjin99 committed Feb 25, 2024
1 parent faf7ca2 commit 6240eba
Show file tree
Hide file tree
Showing 2 changed files with 52 additions and 54 deletions.
12 changes: 5 additions & 7 deletions obs3dian/src/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@


def put_images_in_md(
s3: S3, markdown_file_name: str, image_path_generator: Generator[Path, None, None]
s3: S3, markdown_path: Path, image_path_generator: Generator[Path, None, None]
) -> List[Path]:
"""
Generate local images path by using generator and put images to S3.
Expand All @@ -24,10 +24,9 @@ def put_images_in_md(
"""
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
futures = []
for path in image_path_generator:
s3_key = s3.create_s3_key(markdown_file_name, path.name) # create Key
for image_path in image_path_generator:
futures.append(
executor.submit(s3.put_image, path, s3_key)
executor.submit(s3.put_image, markdown_path, image_path)
) # run upload by multithread

put_image_paths: List[Path] = [] # put success image path list
Expand Down Expand Up @@ -66,15 +65,14 @@ def run(markdown_file_path: Path) -> None:
image_path_generator: Generator = generate_local_image_paths(
image_folder_path, markdown_file_path
)
markdown_file_name = markdown_file_path.stem

put_image_paths = put_images_in_md(s3, markdown_file_name, image_path_generator)
put_image_paths = put_images_in_md(s3, markdown_file_path, image_path_generator)

# (image name, S3 URL) to convert link
link_replace_pairs: List[Tuple[str, str]] = []
for image_path in put_image_paths:
image_name = image_path.name
s3_url = s3.get_image_url(markdown_file_name, image_name)
s3_url = s3.get_image_url(markdown_file_path, image_path)
link_replace_pairs.append(
(image_name, s3_url)
) # image name in .md would convert to S3 url
Expand Down
94 changes: 47 additions & 47 deletions obs3dian/src/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,33 +17,44 @@ def __init__(self, profile_name: str, bucket_name: str) -> None:
print("Can't Connect S3")
raise e

def create_s3_key(self, markdown_file_name: str, image_name: str) -> str:
"""
Create key by makrdown file name and image name key is used in S3
Args:
markdown_file_name (str): mark down file name
image_name (str): image file name in markdown
Returns:
str: S3 key
"""
return f"{markdown_file_name}/{image_name}"

def _check_bucket_exist(self) -> bool:
try:
self.s3.head_bucket(Bucket=self.bucket_name)
return True

except ClientError as e:
error_code = e.response["Error"]["Code"]
assert error_code == "404", "Forbidden or Badrequest on S3"
if error_code != "404": # if no permission or bad request raise error
raise e
return False

def put_public_access_policy(self) -> None:
policy = {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "id-1",
"Action": ["s3:GetObject"],
"Effect": "Allow",
"Resource": f"arn:aws:s3:::{self.bucket_name}/*",
"Principal": "*",
}
],
}
try:
self.s3.put_bucket_policy(
Bucket=self.bucket_name, Policy=json.dumps(policy)
)

except Exception as e:
print("Can't allow public acess to bucket")
raise e

def create_bucket(self) -> bool:
if self._check_bucket_exist():
return False
try:
if self._check_bucket_exist(): # If bucket already exists
return False

bucket_name = self.bucket_name
self.s3.create_bucket(
CreateBucketConfiguration={"LocationConstraint": "ap-northeast-2"},
Expand All @@ -63,48 +74,37 @@ def create_bucket(self) -> bool:
return True

except ClientError as e:
print("Error occured in create bucket ", e)
print("Error occured in create bucket")
raise e

def put_public_access_policy(self) -> None:
policy = {
"Version": "2012-10-17",
"Statement": [
{
"Sid": "id-1",
"Action": ["s3:GetObject"],
"Effect": "Allow",
"Resource": f"arn:aws:s3:::{self.bucket_name}/*",
"Principal": "*",
}
],
}
try:
self.s3.put_bucket_policy(
Bucket=self.bucket_name, Policy=json.dumps(policy)
)
# def create_s3_key(self, markdown_file_name: str, image_name: str) -> str:
# """
# Create key by makrdown file name and image name.

except Exception as e:
print("Can't allow public acess to bucket check permission")
raise e
# Args:
# markdown_file_name (str): mark down file name
# image_name (str): image file name in markdown

# Returns:
# str: S3 key
# """
# return f"{markdown_file_name}/{image_name}"

def get_image_url(self, markdown_file_name: str, image_name: str) -> str:
def get_image_url(self, markdown_path: Path, image_path: Path) -> str:
region = self.session.region_name
key = self.create_s3_key(markdown_file_name, image_name)
s3_url = (
f"https://{self.bucket_name}.s3.{region}.amazonaws.com/{parse.quote(key)}"
)
key = markdown_path / image_path
s3_url = f"https://{self.bucket_name}.s3.{region}.amazonaws.com/{parse.quote(str(key))}"
return s3_url

def put_image(self, file_path: Path, key: str) -> Path:
def put_image(self, markdown_path: Path, image_path: Path) -> Path:
try:
self.s3.put_object(
Bucket=self.bucket_name,
Body=file_path.open("rb"),
Key=key,
Body=image_path.open("rb"),
Key=str(markdown_path / image_path),
)
return file_path
return image_path

except ClientError as e:
print(f"Error Occured in uploading {file_path}")
print(f"Error Occured in uploading {image_path}")
raise e

0 comments on commit 6240eba

Please sign in to comment.