Skip to content

Commit

Permalink
Add description
Browse files Browse the repository at this point in the history
  • Loading branch information
thinkjin99 committed Feb 23, 2024
1 parent 676c033 commit 81e67d1
Show file tree
Hide file tree
Showing 3 changed files with 65 additions and 22 deletions.
2 changes: 1 addition & 1 deletion main.py
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ def config():
def run(user_input_path: Path):
"""
Run obs3dian main command, get image local file paths from md file.
After extracting image paths upload images to s3 and replace all file links in .md to S3 links.
After extracting image paths, it uploads images to s3 and replaces all file links in .md to S3 links.
Output.md will be wrriten under output folder and it's image links would have been replaced to S3 links
Args:
Expand Down
80 changes: 61 additions & 19 deletions src/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,42 +7,84 @@
from typing import Generator, Callable, List


def create_s3_key(markdown_file_name: str, path: Path) -> str:
return f"{markdown_file_name}/{path.name}"
def create_s3_key(markdown_file_name: str, image_name: str) -> str:
"""
Create key by makrdown file name and image name key is used in S3
Args:
markdown_file_name (str): mark down file name
image_name (str): image file name in markdown
Returns:
str: S3 key
"""
return f"{markdown_file_name}/{image_name}"


def put_images_in_md(
s3: S3, markdown_file_name: str, path_generator: Generator[Path, None, None]
s3: S3, markdown_file_name: str, image_path_generator: Generator[Path, None, None]
) -> List[Path]:
"""
Generate local images path by using generator and put images to S3.
Return sccessfully put image paths
Args:
s3 (S3): instance to control S3
markdown_file_name (str): makrdown file name
image_path_generator (Generator[Path, None, None]): yield image paths to upload
Returns:
List[Path]: sccessfully put image paths
"""
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as executor:
futures = []
for path in path_generator: # s3 key
s3_key = create_s3_key(markdown_file_name, path)
futures.append(executor.submit(s3.put_image, path, s3_key))
for path in image_path_generator:
s3_key = create_s3_key(markdown_file_name, path.name) # create Key
futures.append(
executor.submit(s3.put_image, path, s3_key)
) # run upload by multithread

links_map = []
put_image_paths: List[Path] = [] # put success image path list
for future in concurrent.futures.as_completed(futures):
try:
links_map.append(future.result())
put_image_paths.append(future.result())
except Exception as e:
print("Erorr Occured: ", str(e))
continue

return links_map
return put_image_paths


def create_obs3dian_runner(s3: S3, output_folder_path: Path) -> Callable:
def run_obs3dian(markdown_file_path: Path) -> None:
path_generator: Generator = generate_local_paths(markdown_file_path)
"""
Create runner fucntion object
S3 controller and ouput_folder_path would not change before config
Args:
s3 (S3): S3 controller
output_folder_path (Path): output folder path
Returns:
Callable: runner
"""

def run(markdown_file_path: Path) -> None:
"""
Run obs3dian command extract image paths and replace them by S3 URLs.
Args:
markdown_file_path (Path): mark down file path
"""
image_path_generator: Generator = generate_local_paths(markdown_file_path)
markdown_file_name = markdown_file_path.stem

uploaded_image_paths = put_images_in_md(s3, markdown_file_name, path_generator)
put_image_paths = put_images_in_md(s3, markdown_file_name, image_path_generator)

link_replace_map = [
(path.name, s3.get_s3_url(create_s3_key(markdown_file_name, path)))
for path in uploaded_image_paths
]
link_replace_pairs = [
(path.name, s3.get_s3_url(create_s3_key(markdown_file_name, path.name)))
for path in put_image_paths
] # get (local image path, S3 URL) to convert link

write_md_file(markdown_file_path, output_folder_path, link_replace_map)
write_md_file(markdown_file_path, output_folder_path, link_replace_pairs)
return

return run_obs3dian
return run
5 changes: 3 additions & 2 deletions src/s3.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def get_s3_url(self, key: str) -> str:
)
return s3_url

def put_image(self, file_path: Path, key: str) -> Path | None:
def put_image(self, file_path: Path, key: str) -> Path:
try:
self.s3.put_object(
Bucket=self.bucket_name,
Expand All @@ -92,4 +92,5 @@ def put_image(self, file_path: Path, key: str) -> Path | None:
return file_path

except ClientError as e:
print("Error Occured in uploading...\n", e)
print(f"Error Occured in uploading {file_path}")
raise e

0 comments on commit 81e67d1

Please sign in to comment.