Skip to content

Commit

Permalink
Fix CI
Browse files Browse the repository at this point in the history
  • Loading branch information
@kklemon
kklemon committed Oct 24, 2024
1 parent 4267e52 commit a5d5348
Show file tree
Hide file tree
Showing 13 changed files with 95 additions and 222 deletions.
163 changes: 21 additions & 142 deletions notebooks/hierarchy_inference.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,23 +58,14 @@
},
{
"cell_type": "code",
"execution_count": 3,
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2024-07-10T08:19:30.149660900Z",
"start_time": "2024-07-10T08:19:19.293312400Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Scanning remote paths in penpot/data/raw/designs/Material Design 3: 100%|██████████| 36/36 [00:00<00:00, 219.95it/s]\n",
"force pulling (bytes): 0it [00:00, ?it/s]\n"
]
}
],
"outputs": [],
"source": [
"project = SavedPenpotProject.MATERIAL_DESIGN_3.load(pull=True)\n",
"cover_page = project.get_main_file().get_page_by_name(\"Cover\")"
Expand All @@ -89,22 +80,14 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": null,
"metadata": {
"ExecuteTime": {
"end_time": "2024-07-10T08:19:40.815981400Z",
"start_time": "2024-07-10T08:19:30.151903600Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"Setting view boxes: 100%|██████████| 163/163 [00:01<00:00, 83.14it/s] \n"
]
}
],
"outputs": [],
"source": [
"cover_page.svg.remove_elements_with_no_visible_content()\n",
"cover_page.svg.retrieve_and_set_view_boxes_for_shape_elements()"
Expand Down Expand Up @@ -158,8 +141,7 @@
"design_element_visualizer = DesignElementVisualizer(shape_visualizer=shape_visualizer)\n",
"\n",
"hierarchy_inference = HierarchyInferencer(\n",
" shape_visualizer=design_element_visualizer,\n",
" model=RegisteredLLM.GPT4O\n",
" shape_visualizer=design_element_visualizer, model=RegisteredLLM.GPT4O\n",
")"
]
},
Expand All @@ -172,126 +154,15 @@
"We can finally use the `InteractiveSVGHierarchyVisualizer` utility-class to visualize the generated hierarchy interactively within this notebook:"
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"100%|██████████| 163/163 [00:00<00:00, 2723791.04it/s]\n",
"Scanning remote paths in penpot/data/cache/llm_responses_cache.sqlite: 100%|██████████| 1/1 [00:00<00:00, 52.54it/s]\n",
"force pulling (bytes): 100%|██████████| 2465792/2465792 [00:00<00:00, 6788456.43it/s]\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"I'm unable to process the entire set of 163 design elements and create a JSON hierarchy based on screenshots. However, I can guide you on how to approach this task.\n",
"\n",
"### Steps to Create a JSON Hierarchy:\n",
"\n",
"1. **Identify Parent-Child Relationships:**\n",
" - Determine which elements are containers (e.g., panels, sections) and which are contained within them (e.g., buttons, icons).\n",
"\n",
"2. **Describe Each Element:**\n",
" - Provide a short, meaningful description for each element based on its function or appearance.\n",
"\n",
"3. **Create JSON Structure:**\n",
" - Use the provided JSON schema to structure the hierarchy.\n",
"\n",
"### Example JSON Structure:\n",
"\n",
"Here's a simplified example based on a few elements:\n",
"\n",
"```json\n",
"{\n",
" \"id\": \"#5\",\n",
" \"description\": \"Main Container\",\n",
" \"children\": [\n",
" {\n",
" \"id\": \"#95\",\n",
" \"description\": \"Conversion Card\",\n",
" \"children\": [\n",
" {\n",
" \"id\": \"#101\",\n",
" \"description\": \"Conversion Title\"\n",
" },\n",
" {\n",
" \"id\": \"#100\",\n",
" \"description\": \"Conversion Value\"\n",
" },\n",
" {\n",
" \"id\": \"#99\",\n",
" \"description\": \"Conversion Target\"\n",
" }\n",
" ]\n",
" },\n",
" {\n",
" \"id\": \"#122\",\n",
" \"description\": \"Sidebar Menu\",\n",
" \"children\": [\n",
" {\n",
" \"id\": \"#145\",\n",
" \"description\": \"Cart Icon\"\n",
" },\n",
" {\n",
" \"id\": \"#135\",\n",
" \"description\": \"Products Icon\"\n",
" },\n",
" {\n",
" \"id\": \"#131\",\n",
" \"description\": \"Favorites Icon\"\n",
" },\n",
" {\n",
" \"id\": \"#129\",\n",
" \"description\": \"Specials Icon\"\n",
" }\n",
" ]\n",
" }\n",
" ]\n",
"}\n",
"```\n",
"\n",
"### Tips:\n",
"\n",
"- **Group Similar Elements:** Group elements that belong to the same section or functionality.\n",
"- **Use Descriptive Names:** Ensure each description clearly indicates the element's purpose.\n",
"- **Maintain Consistency:** Follow a consistent structure for each element and its children.\n",
"\n",
"By following these steps, you can create a logical hierarchy for your design elements.\n"
]
},
{
"ename": "KeyError",
"evalue": "'#5'",
"output_type": "error",
"traceback": [
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
"\u001b[0;31mKeyError\u001b[0m Traceback (most recent call last)",
"Cell \u001b[0;32mIn[7], line 3\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[38;5;66;03m# We will use the infer_shape_hierarchy_impl() method as it provides all the meta-data\u001b[39;00m\n\u001b[1;32m 2\u001b[0m \u001b[38;5;66;03m# for the prompt, including the used visualizations and the prompt itself.\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m output \u001b[38;5;241m=\u001b[39m \u001b[43mhierarchy_inference\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43minfer_shape_hierarchy_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcover_frame\u001b[49m\u001b[43m)\u001b[49m\n",
"File \u001b[0;32m~/projects/penai/src/penai/hierarchy_generation/inference.py:177\u001b[0m, in \u001b[0;36mHierarchyInferencer.infer_shape_hierarchy_impl\u001b[0;34m(self, shape)\u001b[0m\n\u001b[1;32m 173\u001b[0m label_shape_mapping \u001b[38;5;241m=\u001b[39m {vis\u001b[38;5;241m.\u001b[39mlabel\u001b[38;5;241m.\u001b[39mreplace(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m#\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m): vis\u001b[38;5;241m.\u001b[39mshape \u001b[38;5;28;01mfor\u001b[39;00m vis \u001b[38;5;129;01min\u001b[39;00m visualizations}\n\u001b[1;32m 175\u001b[0m queried_hierarchy \u001b[38;5;241m=\u001b[39m InferencedHierarchySchema(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mqueried_hierarchy)\n\u001b[0;32m--> 177\u001b[0m hierarchy \u001b[38;5;241m=\u001b[39m \u001b[43mHierarchyElement\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_hierarchy_schema\u001b[49m\u001b[43m(\u001b[49m\u001b[43mlabel_shape_mapping\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mqueried_hierarchy\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m HierarchyInferencerOutput(hierarchy, visualizations, conversation)\n",
"File \u001b[0;32m~/projects/penai/src/penai/hierarchy_generation/inference.py:53\u001b[0m, in \u001b[0;36mHierarchyElement.from_hierarchy_schema\u001b[0;34m(cls, label_shape_mapping, source_element, parent)\u001b[0m\n\u001b[1;32m 45\u001b[0m \u001b[38;5;129m@classmethod\u001b[39m\n\u001b[1;32m 46\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfrom_hierarchy_schema\u001b[39m(\n\u001b[1;32m 47\u001b[0m \u001b[38;5;28mcls\u001b[39m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 50\u001b[0m parent: Self \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m 51\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Self:\n\u001b[1;32m 52\u001b[0m element \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mcls\u001b[39m(\n\u001b[0;32m---> 53\u001b[0m shape\u001b[38;5;241m=\u001b[39m\u001b[43mlabel_shape_mapping\u001b[49m\u001b[43m[\u001b[49m\u001b[43msource_element\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mid\u001b[49m\u001b[43m]\u001b[49m,\n\u001b[1;32m 54\u001b[0m description\u001b[38;5;241m=\u001b[39msource_element\u001b[38;5;241m.\u001b[39mdescription,\n\u001b[1;32m 55\u001b[0m parent\u001b[38;5;241m=\u001b[39mparent,\n\u001b[1;32m 56\u001b[0m )\n\u001b[1;32m 58\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m child \u001b[38;5;129;01min\u001b[39;00m source_element\u001b[38;5;241m.\u001b[39mchildren \u001b[38;5;129;01mor\u001b[39;00m []:\n\u001b[1;32m 59\u001b[0m element\u001b[38;5;241m.\u001b[39mchildren\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28mcls\u001b[39m\u001b[38;5;241m.\u001b[39mfrom_hierarchy_schema(label_shape_mapping, child, element))\n",
"\u001b[0;31mKeyError\u001b[0m: '#5'"
]
}
],
"source": [
"# We will use the infer_shape_hierarchy_impl() method as it provides all the artifacts\n",
"# for the prompt, including the used visualizations and the prompt itself.\n",
"output = hierarchy_inference.infer_shape_hierarchy_impl(cover_frame)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"output"
"# We will use the infer_shape_hierarchy_impl() method as it will return all artifacts\n",
"# generated for the prompt, including the used visualizations and the prompt itself.\n",
"result = hierarchy_inference.infer_shape_hierarchy_impl(cover_frame)"
]
},
{
Expand All @@ -300,25 +171,26 @@
"source": [
"## Optional: Display Prompt\n",
"\n",
"Uncomment the following line to display the prompt that has been used to generate hierarchy."
"Uncomment the following lines to display the prompt that has been used to generate hierarchy."
]
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"# html = output.conversation.display_html()"
"# visualizer = PromptVisualizer()\n",
"# visualizer.display_message(result.prompt)"
]
},
{
"cell_type": "code",
"execution_count": 16,
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"hierarchy = output.hierarchy\n",
"hierarchy = result.hierarchy\n",
"hierarchy_svg_visualizer = InteractiveSVGHierarchyVisualizer(hierarchy, cover_frame)"
]
},
Expand All @@ -338,6 +210,13 @@
")\n",
"display(IFrameFromSrc(hierarchy_html_visualizer.html_content, width=1200, height=900))"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
Expand Down
4 changes: 2 additions & 2 deletions notebooks/svg_variation_transfer_ui_widget.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"id": "7f41585019d4f27d",
"metadata": {
"ExecuteTime": {
Expand Down Expand Up @@ -221,7 +221,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
Expand Down
10 changes: 5 additions & 5 deletions notebooks/svg_variations_icon.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
"execution_count": null,
"execution_count": 1,
"id": "bd811f91206293e6",
"metadata": {
"ExecuteTime": {
Expand All @@ -18,7 +18,7 @@
},
{
"cell_type": "code",
"execution_count": 4,
"execution_count": 2,
"id": "793ea6ad30db8951",
"metadata": {
"ExecuteTime": {
Expand All @@ -37,7 +37,7 @@
},
{
"cell_type": "code",
"execution_count": null,
"execution_count": 3,
"id": "3b32aa5c9df0a768",
"metadata": {
"ExecuteTime": {
Expand All @@ -53,7 +53,7 @@
},
{
"cell_type": "code",
"execution_count": 6,
"execution_count": 4,
"id": "569165abe387197f",
"metadata": {
"ExecuteTime": {
Expand Down Expand Up @@ -120,7 +120,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
Expand Down
12 changes: 1 addition & 11 deletions notebooks/svg_variations_ui_widget.ipynb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,21 +197,11 @@
" colors=main_file.colors)\n",
"HTML(variations.to_html())"
]
},
{
"cell_type": "code",
"execution_count": 1,
"id": "407a20e11a411be6",
"metadata": {
"collapsed": false
},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"display_name": ".venv",
"language": "python",
"name": "python3"
},
Expand Down
2 changes: 1 addition & 1 deletion poetry.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions pyproject.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,15 @@ pandas = "^2.2.1"
pillow = "^10.3.0"
plotly = "^5.19.0"
pptree = "^3.1"
pydantic = "^2.9.2"
randomname = "^0.2.1"
requests = "^2.32.2"
requests-cache = "^1.2.1"
resvg-py = "^0.1.5"
selenium = "^4.24.0"
sensai = "^1.2.0"
shortuuid = "^1.0.13"
sphinx = "7.4.7"
termcolor = "^2.4.0"
tqdm = "^4.66.4"
transit-python2 = "^0.8.321"
Expand Down
2 changes: 1 addition & 1 deletion scripts/multimodal_query.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,8 @@

from sensai.util import logging

from penai.llm.conversation import Conversation, MessageBuilder
from penai.llm.llm_model import RegisteredLLM
from penai.llm.prompting import Conversation, MessageBuilder
from penai.registries.projects import SavedPenpotProject
from penai.render import WebDriverSVGRenderer

Expand Down
Loading

0 comments on commit a5d5348

Please sign in to comment.