Modified hierarchy logic

Author: paf-triarii

m_aux/outputs.py

@@ -1,4 +1,5 @@
 import os
+import re
 import shutil
 
 
@@ -40,3 +41,12 @@ def prepare_output_folder(folder_path):
         print(f"Folder '{folder_path}' was emptied and recreated.")
     except Exception as e:
         print(f"An error occurred while preparing the folder: {e}")
+
+
+def normalize_string(name):
+    """Normalizes names for files and directories."""
+    # Replace spaces with dashes, remove special characters, trim, and lowercase
+    name = re.sub(r"\s+", "-", name)  # Spaces to dashes
+    name = re.sub(r"[^\w\-]", "", name)  # Remove non-word characters except dashes
+    name = name.strip("-")  # Trim leading and trailing dashes
+    return name.lower()

m_parse/block_models.py

@@ -81,6 +81,11 @@ class EmbedBlock(BaseModel):
     url: str
 
 
+class BulletedListItemBlock(BaseModel):
+    rich_text: List[RichText]
+    color: str
+
+
 class Block(BaseModel):
     object: str
     id: str
@@ -96,6 +101,7 @@ class Block(BaseModel):
     image: Optional[ImageBlock] = None
     bookmark: Optional[BookmarkBlock] = None
     embed: Optional[EmbedBlock] = None
+    bulleted_list_item: Optional[BulletedListItemBlock] = None
     # Add other fields and types as necessary
     dynamic_parents: Dict[str, ParentReference] = Field(default_factory=dict)
 

m_parse/markdown_processing.py

@@ -2,6 +2,7 @@
 from m_parse.block_models import (
     Block,
     BookmarkBlock,
+    BulletedListItemBlock,
     ChildPageBlock,
     CodeBlock,
     EmbedBlock,
@@ -15,6 +16,7 @@
     validate_block,
 )
 from m_parse.markdown_processing_helpers import (
+    markdown_bullet,
     markdown_code_block,
     markdown_convert_paragraph_styles,
     markdown_headings,
@@ -232,7 +234,7 @@ def parse_child_page(block: ChildPageBlock):
         )
     )
     page_processed_blocks.append(
-        parsing_block_return(block.id, changelog, block.type, path_hierarchy)
+        parsing_block_return(block.id, changelog, "changelog", path_hierarchy)
     )
     return page_processed_blocks
 
@@ -288,6 +290,21 @@ def parse_embed(block: Block) -> dict:
     )
 
 
+@validate_block(BulletedListItemBlock)
+def parse_bulleted_list_item(block: Block, indent_level: int = 0) -> dict:
+    """Parses a bulleted list item block into Markdown format, considering indentation and
+    styles."""
+    bullet_items = []
+    for rich_text_item in block.bulleted_list_item.rich_text:
+        content = rich_text_item.plain_text
+        annotations = rich_text_item.annotations
+        bullet_items.append(markdown_bullet(content, annotations, indent=indent_level))
+
+    md = "\n".join(bullet_items)
+
+    return parsing_block_return(block.id, md, block.type, calculate_path_on_hierarchy(block))
+
+
 @validate_block(LinkToPageBlock)
 def parse_link_to_page(block: LinkToPageBlock):
     # TODO: Implement this function

m_parse/markdown_processing_helpers.py

@@ -180,3 +180,13 @@ def markdown_image(image_url: str, caption: str = "") -> str:
     - str: The formatted Markdown image link string.
     """
     return f"![{caption}]({image_url})"
+
+
+def markdown_bullet(content: str, annotations: dict, indent: int = 0) -> str:
+    """Generates a markdown bullet list item with optional indentation and styles."""
+    # Convert styles for the bullet content
+    styled_content = markdown_convert_paragraph_styles(content, annotations)
+    # Calculate the indentation spaces
+    indent_spaces = "  " * indent
+    # Combine to form the markdown bullet item
+    return f"{indent_spaces}- {styled_content}"

m_search/notion_blocks.py

@@ -14,6 +14,13 @@ def fetch_and_process_block_hierarchy(root_block_id):
     - list: A list of all processed blocks, each with added parent hierarchy information.
     """
     processed_blocks = []
+    root_block = fetch_block_details(root_block_id)
+    root_block_parent = root_block.get("parent", None)
+    root_block_parent_id = (
+        (root_block_parent.get("block_id") or root_block_parent.get("page_id")).strip()
+        if root_block_parent
+        else None
+    )
 
     def process_block(block_id, parent_hierarchy=[]):
         """Recursively processes a block and its children, adding parent hierarchy information.
@@ -28,7 +35,9 @@ def process_block(block_id, parent_hierarchy=[]):
             return
 
         # Add parent hierarchy information to the current block
-        add_parent_hierarchy(current_block, parent_hierarchy.copy())
+        add_parent_hierarchy(
+            current_block, parent_hierarchy.copy(), root_block_id, root_block_parent_id
+        )
 
         # Add the processed block to the list
         processed_blocks.append(current_block)
@@ -49,7 +58,9 @@ def process_block(block_id, parent_hierarchy=[]):
     return processed_blocks
 
 
-def add_parent_hierarchy(block, parent_hierarchy=[]):
+def add_parent_hierarchy(
+    block, parent_hierarchy=[], root_block_id=None, root_block_parent_id=None
+):
     """Adds parent hierarchy identifiers to a block, ensuring no duplications and starting labeling
     from c_parent_1.
 
@@ -62,6 +73,9 @@ def add_parent_hierarchy(block, parent_hierarchy=[]):
     parent_id = parent_id.strip() if parent_id else None
     parent_type = block.get("type")
 
+    if root_block_id and block["id"] != root_block_id and root_block_parent_id:
+        parent_hierarchy.insert(0, {"block_id": root_block_parent_id, "type": "root"})
+
     # Normalize block_id before comparison
     normalized_parent_id = parent_id.replace("-", "") if parent_id else None
 

m_write/notion_processed_blocks.py

@@ -0,0 +1,40 @@
+"""Module for processing and writing Notion blocks to Markdown files."""
+
+import os
+from m_write.write_helpers import ensure_dir, write_md_file, find_page_title
+from m_aux.outputs import normalize_string
+from m_aux.pretty_print import pretty_print
+
+# def process_blocks(blocks, root_dir, parent_id=None):
+#     """Processes and writes blocks to the appropriate .md files and directories, with improvements."""
+#     for block in blocks:
+#         # Handle 'child_page' blocks differently to use their title for naming
+#         if block['type'] == 'child_page':
+#             page_title = find_page_title(blocks, block['id'])
+#             # Normalize the page title for use in file and directory names
+#             normalized_title = normalize_string(page_title)
+#             output_path = os.path.join(root_dir, block['path'].replace('/', os.sep), normalized_title)
+#             ensure_dir(output_path)  # Ensure the directory for the page exists
+#             output_file = os.path.join(output_path, normalized_title + ".md")
+#         else:
+#             # Find the title of the parent page for non-'child_page' blocks
+#             if parent_id:
+#                 parent_title = find_page_title(blocks, parent_id)
+#                 normalized_title = normalize_string(parent_title)
+#                 output_path = os.path.join(root_dir, block['path'].replace('/', os.sep), normalized_title)
+#             else:
+#                 output_path = os.path.join(root_dir, block['path'].replace('/', os.sep))
+#             ensure_dir(output_path)
+#             output_file = os.path.join(output_path, normalize_string(block['md'][:50]) + ".md")  # Use first 50 chars of md as filename
+        
+#         write_md_file(output_file, block['md'])
+
+#         # Recursively process subpages for 'child_page' blocks
+#         if block['type'] == 'child_page':
+#             subpages = [b for b in blocks if b['path'].startswith(block['path']) and b['id'] != block['id']]
+#             if subpages:
+#                 process_blocks(subpages, output_path, block['id'])
+
+
+def process_blocks(blocks, root_dir):
+  pretty_print(blocks, "Processed blocks")

m_write/write_helpers.py

@@ -0,0 +1,41 @@
+import os
+from m_aux.outputs import normalize_string
+
+def ensure_dir(directory):
+    """Ensures that a directory exists."""
+    if not os.path.exists(directory):
+        os.makedirs(directory)
+
+def write_md_file(file_path, content):
+    """Writes Markdown content to a file."""
+    with open(file_path, 'w', encoding='utf-8') as md_file:
+        md_file.write(content)
+
+def find_page_title(blocks, block_id):
+    """Finds a page title given a block ID."""
+    for block in blocks:
+        if block['id'] == block_id and 'md' in block:
+            return normalize_string(block['md'].strip('# ').strip())
+    return None
+
+def get_md_content(blocks, block_id):
+    """Fetches the Markdown content for a block by its ID."""
+    for block in blocks:
+        if block['id'] == block_id:
+            return block['md']
+    return None
+
+def is_page(blocks, block_id):
+    """Checks if a block is of type 'child_page'."""
+    for block in blocks:
+        if block['id'] == block_id:
+            return block['type'] == 'child_page'
+    return False
+
+def get_pages(blocks):
+    """Extracts child pages from blocks and sorts them by the depth of their path."""
+    # Filter only child pages
+    pages = [block for block in blocks if block['type'] == 'child_page']
+    # Sort pages by path length, then by path value for equal lengths
+    pages.sort(key=lambda x: (len(x['path'].split('/')), x['path']))
+    return pages

main.py

@@ -13,6 +13,7 @@
     get_all_children_blocks,
 )
 from m_search.notion_pages import fetch_page_details
+from m_write.notion_processed_blocks import process_blocks
 
 
 def main():
@@ -68,9 +69,10 @@ def main():
     # children_blocks = get_all_children_blocks(args.page_id)
     # pretty_print(children_blocks)
     blocks = fetch_and_process_block_hierarchy(args.page_id)
-    pretty_print(blocks)
+    pretty_print(blocks, "Fetched blocks")
     processed_blocks = dispatch_blocks_parsing(blocks)
-    pretty_print(processed_blocks)
+    # pretty_print(processed_blocks)
+    process_blocks(processed_blocks, args.outputs_dir)
     # pretty_print(children_blocks)
     # page_details = fetch_page_details(notion, args.page_id)
     # pretty_print(page_details)