Skip to content

Commit e355cf1

Browse files
HarrySu123HarrySu123
committed
split validating $, $$ placement phase into parts to increase accuracy per question
1 parent fa6dd2a commit e355cf1

File tree

1 file changed

+57
-43
lines changed

1 file changed

+57
-43
lines changed

‎conversion2025/mathpix_to_llm_to_in2lambda_to_JSON.ipynb‎

Lines changed: 57 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,7 @@
318318
" 4. Look through the entire markdown:\n",
319319
" - Do not neglect any images, figures, or other media mentioned in the question, do not alter or neglect the alt text and the image URL.\n",
320320
" - Leave the Image links and alt text within the question/solution, but also make a copy and place it into the `images` field.\n",
321-
" - Identify full Questions, place it into question_content\n",
321+
" - Identify full Questions, place it into question_content, becareful to not Include the solution in the question.\n",
322322
" - Identify the full Worked Solution for each full Question.\n",
323323
" - If the Worked Solution is not found, try to find the Answers associated with it instead.\n",
324324
" - If Worked Solution or Answers are found, place it into the solution_content. Otherwise leave as empty string, \"\".\n",
@@ -404,7 +404,7 @@
404404
" - Identify the stem and parts of the question, the parts may be obvious to find, like \"a)...\", \"b)...\", etc., or they could be implied by the question itself. All question must have at least one part, if there is only one part. :\n",
405405
" 1. The stem should be placed into the \"content\" field. Text in this field should be valid in the Milkdown editor. \n",
406406
" 2. the parts of the question (subquestions) should be placed into the \"parts\" field. Text in this field should be valid under Lexdown.\n",
407-
" 3. for each part, identify the worked solution/answer and place it into the \"parts_solutions\" field, if not found, leave as empty string, \"\". Text in this field should be valid under Lexdown.\n",
407+
" 3. for each part, carefully identify the worked solution/answer associated with it and place it into the \"parts_solutions\" field, if not found, leave as empty string, \"\". Text in this field should be valid under Lexdown.\n",
408408
" 5. Output only a valid, plain, raw JSON string matching the schema above, ready to parse immediately, with no code fence or extra text. Use plain newlines (not escaped as `\\n`).\n",
409409
" 6. The Text inside the JSON should be in Lexdown:\n",
410410
" 1. preserving all LaTeX math delimiters (`$...$` and `$$...$$`) and all formatting exactly as in the input, without paraphrasing, summarizing, or simplifying any mathematical expressions or formulas.\n",
@@ -442,14 +442,12 @@
442442
" \"\"\"\n",
443443
" # Initialize the output parser with the Tutorial schema.\n",
444444
" parser = PydanticOutputParser(pydantic_object=Set_Question)\n",
445-
"\n",
446-
"\n",
447445
" \n",
448446
" questions_in_parts = []\n",
449-
" for question in questions_dict[\"questions\"]:\n",
447+
" for question_idx, question in enumerate(questions_dict[\"questions\"]):\n",
450448
" passed = False\n",
451449
"\n",
452-
" for idx in range(3):\n",
450+
" for attempt_idx in range(3):\n",
453451
"\n",
454452
" # Construct the prompt, appending the parser's format instructions.\n",
455453
" prompt = f\"\"\"\n",
@@ -458,7 +456,7 @@
458456
"\n",
459457
" {llm_task_seperate_parts}\n",
460458
"\n",
461-
" Input JSON:\n",
459+
" Input Dictionary:\n",
462460
" ```JSON\n",
463461
" {question}\n",
464462
" ```\n",
@@ -476,17 +474,17 @@
476474
" try:\n",
477475
" # Parse the response using the output parser.\n",
478476
" parsed_output = parser.parse(response.content)\n",
479-
" print(f\"LLM response successfully parsed question {idx}.\")\n",
477+
" print(f\"LLM response successfully parsed question {question_idx}.\")\n",
480478
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
481479
" passed = True\n",
482480
" break\n",
483481
" except Exception as e:\n",
484-
" print(\"Error parsing LLM response as JSON:\")\n",
485-
" print(\"Retrying...\")\n",
482+
" print(f\"Error parsing LLM response as JSON for question {question_idx}:\")\n",
483+
" print(f\"Retrying... Attempt No.{attempt_idx + 1}\")\n",
486484
" time.sleep(2)\n",
487485
"\n",
488486
" if not passed:\n",
489-
" raise Exception(\"Failed to parse LLM response as JSON after multiple attempts.\")\n",
487+
" raise Exception(f\"Failed to parse LLM response as JSON after multiple attempts for question {question_idx}.\")\n",
490488
" \n",
491489
" questions_in_parts.append(parsed_output)\n",
492490
" \n",
@@ -515,7 +513,7 @@
515513
"outputs": [],
516514
"source": [
517515
"llm_task_expression_check = r\"\"\"\n",
518-
" Look inside the JSON object's `questions`, specifically the `content`, `parts`, and `parts_solutions` fields. Ensure that the JSON content follows these rules:\n",
516+
" Look inside the structure, specifically the `content`, `parts`, and `parts_solutions` fields. Ensure that the JSON content follows these rules:\n",
519517
" 1. No extra escaping: The JSON string must contain no literal `\\\\n`, `\\\\\\\\`, or unnecessary escape sequences unless they are explicitly present in the original input text.\n",
520518
" 2. Careful to make the distinction between inline and display math, i.e. do not mess up the use of `$` and `$$`.\n",
521519
" 3. Math delimiters: All mathematical expressions must be fully enclosed within math delimiters — use `$...$` for inline math, and `$$...$$` for display math.\n",
@@ -555,41 +553,57 @@
555553
" dict: A dictionary containing the keys \"name\" and \"exercise\".\n",
556554
" If parsing fails, returns None.\n",
557555
" \"\"\"\n",
558-
" json_string = json.dumps(validated_dict, indent=2)\n",
559-
" \n",
560-
" # prompt to let llm validate the JSON.\n",
561-
" validation_prompt = f\"\"\"\n",
562-
" {llm_task_expression_check}\n",
563-
"\n",
564-
" Input JSON:\n",
565-
" ```json\n",
566-
" {json_string}\n",
567-
" ```\n",
568-
" return the JSON with the content fixed if needed.\n",
569-
" \"\"\"\n",
556+
" parser = PydanticOutputParser(pydantic_object=Set_Question)\n",
570557
"\n",
571-
" parser = PydanticOutputParser(pydantic_object=Set)\n",
558+
" questions_in_parts = []\n",
559+
" for question_idx, question in enumerate(validated_dict[\"questions\"]):\n",
560+
" passed = False\n",
572561
" \n",
573-
" # loop 3 times to ensure robustness.\n",
574-
" for i in range(3):\n",
575-
" \n",
576-
" # Call the LLM\n",
577-
" response = llm.invoke(validation_prompt)\n",
562+
" # loop 3 times to ensure robustness.\n",
563+
" for attempt_idx in range(3):\n",
564+
" # prompt to let llm validate the JSON.\n",
565+
" validation_prompt = f\"\"\"\n",
566+
" Your task is to extract a JSON with the following structure exactly:\n",
567+
" {parser.get_format_instructions()}\n",
578568
"\n",
579-
" # Debug: print the raw LLM response\n",
580-
" # print(\"Raw LLM Response:\")\n",
581-
" # print(response)\n",
569+
" {llm_task_expression_check}\n",
582570
"\n",
583-
" try:\n",
584-
" # Parse the response using the output parser.\n",
585-
" parsed_output = parser.parse(response.content)\n",
586-
" print(\"LLM response successfully parsed as JSON with valid $$.\")\n",
587-
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
588-
" return parsed_output.model_dump()\n",
589-
" except Exception as e:\n",
590-
" print(\"Error parsing textdown LLM response as JSON:\")\n",
591-
" print(\"Retrying...\")\n",
592-
" time.sleep(2)"
571+
" Input Dictionary:\n",
572+
" ```json\n",
573+
" {question}\n",
574+
" ```\n",
575+
" return the JSON with the content fixed if needed.\n",
576+
" \"\"\"\n",
577+
"\n",
578+
" # Call the LLM\n",
579+
" response = llm.invoke(validation_prompt)\n",
580+
"\n",
581+
" # Debug: print the raw LLM response\n",
582+
" # print(\"Raw LLM Response:\")\n",
583+
" # print(response)\n",
584+
"\n",
585+
" try:\n",
586+
" # Parse the response using the output parser.\n",
587+
" parsed_output = parser.parse(response.content)\n",
588+
" print(f\"LLM response successfully parsed as JSON with valid $$ for question {question_idx}.\")\n",
589+
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
590+
" passed = True\n",
591+
" break\n",
592+
" return parsed_output.model_dump()\n",
593+
" except Exception as e:\n",
594+
" print(f\"Error parsing textdown LLM response as JSON for question {question_idx}:\")\n",
595+
" print(\"Retrying... Attempt No.\", attempt_idx + 1)\n",
596+
" time.sleep(2)\n",
597+
" \n",
598+
" if not passed:\n",
599+
" raise Exception(f\"Failed to parse LLM response as JSON after multiple attempts for question {question_idx}.\")\n",
600+
" \n",
601+
" questions_in_parts.append(parsed_output)\n",
602+
" return Set(\n",
603+
" name=validated_dict[\"name\"],\n",
604+
" year=validated_dict[\"year\"],\n",
605+
" questions=questions_in_parts\n",
606+
" ).model_dump()"
593607
]
594608
},
595609
{

0 commit comments

Comments
 (0)