Skip to content

Commit 55af70e

Browse files
HarrySu123HarrySu123
committed
improved JSON content by letting llm go through it again to correct any syntax mistakes
1 parent 09be7a2 commit 55af70e

File tree

1 file changed

+197
-22
lines changed

1 file changed

+197
-22
lines changed

‎conversion2025/mathpix_to_llm_to_in2lambda_to_JSON.ipynb‎

Lines changed: 197 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,26 @@
307307
" year: str = Field(..., description=\"Year of the set\")\n",
308308
" questions: list[Set_Question] = Field(..., description=\"List of questions in the set\")\n",
309309
"\n",
310-
"def extract_tutorial_questions(doc_page_content: str) -> dict:\n",
310+
"llm_task = \"\"\"\n",
311+
"Please follow these steps carefully:\n",
312+
" 1. You can decide what to call the Set.\n",
313+
" 2. Identify the year of the tutorial, if mentioned. Otherwise, use \"0\".\n",
314+
" 3. Every character should match the original source exactly unless you're instructed to split content into fields.\n",
315+
" 4. Identify the questions in the Input markdown and add them to the \"questions\" list.\n",
316+
" 5. for each question:\n",
317+
" - Title is the only field where you are allowed to name it whatever you seem fit for the question.\n",
318+
" - Identify the content of the question, which will be always visible above the individual parts. This field uses the Milkdown editor.\n",
319+
" - Identify the parts of the question (subquestions) and their worked solutions. The parts could be obvious to find, like \"a)...\", \"b)...\", etc., or they could be implied by the question itself.\n",
320+
" - If the worked solution is not given, leave the worked solution empty.\n",
321+
" - Add the parts of the question (subquestions) and their worked solutions to the \"parts\" and \"parts_solutions\" lists, respectively.\n",
322+
" 6. Output only a valid, plain, raw JSON string matching the schema above, ready to parse immediately, with no code fence or extra text. Use plain newlines (not escaped as `\\n`).\n",
323+
" 7. The Text inside the JSON should be in Lexdown:\n",
324+
" 1. preserving all LaTeX math delimiters (`$...$` and `$$...$$`) and all formatting exactly as in the input, without paraphrasing, summarizing, or simplifying any mathematical expressions or formulas.\n",
325+
" 2. do not remove or collapse blank lines.\n",
326+
" 3. Do not escape characters like `\\n` or `\\\\`.\n",
327+
"\"\"\"\n",
328+
"\n",
329+
"def extract_questions(doc_page_content: str, extra_instruction: str = \"\") -> dict:\n",
311330
" \"\"\"\n",
312331
" Extracts the title and individual questions from a tutorial sheet.\n",
313332
"\n",
@@ -348,18 +367,9 @@
348367
" Your task is to extract a JSON with the following structure exactly:\n",
349368
" {parser.get_format_instructions()}\n",
350369
"\n",
351-
" Please follow these steps carefully:\n",
352-
" 1. Infer a very short and concise title describing the entire Input.\n",
353-
" 2. Identify the year of the tutorial, if mentioned. Otherwise, use \"0\".\n",
354-
" 3. Use the original markdown text exactly as it appears for content, question, parts, and parts_solutions, **preserving all LaTeX math delimiters (`$...$` and `$$...$$`) and all formatting exactly as in the input**, without paraphrasing, summarizing, or simplifying any mathematical expressions or formulas.\n",
355-
" 4. Identify the questions in the Input markdown and add them to the \"questions\" list.\n",
356-
" 5. for each question:\n",
357-
" - Infer the title of the question (only the text, no numbering).\n",
358-
" - Identify the content of the question (no exercise title, no subquestions).\n",
359-
" - Identify the parts of the question (subquestions) and their worked solutions. If the worked solution is not given, leave the worked solution empty.\n",
360-
" - Add the parts of the question (subquestions) and their worked solutions to the \"parts\" and \"parts_solutions\" lists, respectively.\n",
361-
" 6. Output only a valid, plain, raw JSON string matching the schema above, ready to parse immediately, with no extra text, comments, or explanations. Use plain newlines (not escaped as `\\n`).\n",
362-
" 7. The Text inside the JSON should be in Lexdown, preserving all LaTeX math delimiters (`$...$` and `$$...$$`) and all formatting exactly as in the input, without paraphrasing, summarizing, or simplifying any mathematical expressions or formulas. As it will be parsed by KaTex, it should be valid LaTeX.\n",
370+
" {llm_task}\n",
371+
"\n",
372+
" {extra_instruction}\n",
363373
"\n",
364374
" Return the JSON now.\n",
365375
" \"\"\"\n",
@@ -371,36 +381,201 @@
371381
" response = llm.invoke(prompt)\n",
372382
"\n",
373383
" # Debug: print the raw LLM response\n",
374-
" print(\"Raw LLM Response:\")\n",
375-
" print(response)\n",
384+
" # print(\"Raw LLM Response:\")\n",
385+
" # print(response)\n",
376386
"\n",
377387
" try:\n",
378388
" # Parse the response using the output parser.\n",
379389
" parsed_output = parser.parse(response.content)\n",
390+
" print(\"LLM response successfully parsed as JSON.\")\n",
380391
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
381392
" return parsed_output.model_dump()\n",
382393
" except ValidationError as ve:\n",
383394
" print(\"❌ Pydantic Validation Error:\")\n",
384395
" for error in ve.errors():\n",
385396
" print(f\" - {error['loc']}: {error['msg']}\")\n",
386397
" print(\"Raw LLM output:\")\n",
387-
" print(response.content)"
398+
" print(response.content)\n",
399+
" except Exception as e:\n",
400+
" print(\"Error parsing LLM response as JSON:\")\n",
401+
" print(\"Retrying...\")\n",
402+
" time.sleep(2)"
403+
]
404+
},
405+
{
406+
"cell_type": "markdown",
407+
"id": "16",
408+
"metadata": {},
409+
"source": [
410+
"# LLM evaluation of the content of JSON"
388411
]
389412
},
390413
{
391414
"cell_type": "code",
392415
"execution_count": null,
393-
"id": "16",
416+
"id": "17",
394417
"metadata": {},
395418
"outputs": [],
396419
"source": [
397-
"imported_tutorial = extract_tutorial_questions(md_content)"
420+
"def task_rules_obeyed_check(extracted_dict: dict) -> dict:\n",
421+
" \"\"\"\n",
422+
" Extracts the title and individual questions from a tutorial sheet.\n",
423+
" \n",
424+
" Args:\n",
425+
" md_content (str): The content of a set.\n",
426+
" \n",
427+
" Returns:\n",
428+
" dict: A dictionary containing the keys \"name\" and \"exercise\".\n",
429+
" If parsing fails, returns None.\n",
430+
" \"\"\"\n",
431+
" json_string = json.dumps(extracted_dict, indent=2)\n",
432+
" \n",
433+
" # prompt to let llm validate the JSON.\n",
434+
" validation_prompt = f\"\"\"\n",
435+
" You were given the following rules to follow:\n",
436+
" {llm_task}\n",
437+
"\n",
438+
" please make sure that the content of the JSON followed the rules above and return the new JSON.\n",
439+
" {json_string}\n",
440+
" \"\"\"\n",
441+
"\n",
442+
" parser = PydanticOutputParser(pydantic_object=Set)\n",
443+
" # loop 3 times to ensure robustness.\n",
444+
" for i in range(3):\n",
445+
" \n",
446+
" # Call the LLM\n",
447+
" response = llm.invoke(validation_prompt)\n",
448+
"\n",
449+
" # Debug: print the raw LLM response\n",
450+
" # print(\"Raw LLM Response:\")\n",
451+
" # print(response)\n",
452+
"\n",
453+
" try:\n",
454+
" # Parse the response using the output parser.\n",
455+
" parsed_output = parser.parse(response.content)\n",
456+
" print(\"LLM response successfully parsed as validatedJSON.\")\n",
457+
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
458+
" return parsed_output.model_dump()\n",
459+
" except ValidationError as ve:\n",
460+
" print(\"❌ Pydantic Validation Error:\")\n",
461+
" for error in ve.errors():\n",
462+
" print(f\" - {error['loc']}: {error['msg']}\")\n",
463+
" print(\"Raw LLM output:\")\n",
464+
" print(response.content)\n",
465+
" except Exception as e:\n",
466+
" print(\"Error parsing validation LLM response as JSON:\")\n",
467+
" print(\"Retrying...\")\n",
468+
" time.sleep(2)"
398469
]
399470
},
400471
{
401472
"cell_type": "code",
402473
"execution_count": null,
403-
"id": "17",
474+
"id": "18",
475+
"metadata": {},
476+
"outputs": [],
477+
"source": [
478+
"def content_texdown_check(validated_dict: dict) -> dict:\n",
479+
" \"\"\"\n",
480+
" Checks if the content of the JSON is in Texdown format.\n",
481+
" \n",
482+
" Args:\n",
483+
" validated_dict (dict): The validated dictionary from the LLM.\n",
484+
" \n",
485+
" Returns:\n",
486+
" dict: A dictionary containing the keys \"name\" and \"exercise\".\n",
487+
" If parsing fails, returns None.\n",
488+
" \"\"\"\n",
489+
" json_string = json.dumps(validated_dict, indent=2)\n",
490+
" \n",
491+
" # prompt to let llm validate the JSON.\n",
492+
" validation_prompt = f\"\"\"\n",
493+
" Here is a JSON:\n",
494+
" ```json\n",
495+
" {json_string}\n",
496+
" ```\n",
497+
"\n",
498+
" look inside questions, content, parts and parts_solutions, ensure that the content of the JSON follows these rules:\n",
499+
" 1. Ensure the JSON string contains no literal \"\\n\" or \"\\\\\" characters unless explicitly part of the input text.\n",
500+
" 2. All mathematical expressions and formulas must be fully enclosed within matching math delimiters: either inline math `$...$` or display math `$$...$$`.\n",
501+
" 3. Verify all `$$` delimiters are properly opened and closed; no unbalanced or partial math blocks allowed.\n",
502+
" 4. Verify all `$` delimiters are properly opened and closed; inline math should not span multiple lines.\n",
503+
" 5. Preserve all LaTeX formatting, including backslashes and braces, exactly as in the input without adding extra escaping or modifying math commands.\n",
504+
" 6. Blank lines inside math blocks must be preserved as-is.\n",
505+
" 7. Output only a valid JSON string without any additional escaping or characters.\n",
506+
"\n",
507+
"\n",
508+
" return the JSON with the content fixed if needed.\n",
509+
" \"\"\"\n",
510+
"\n",
511+
" parser = PydanticOutputParser(pydantic_object=Set)\n",
512+
" \n",
513+
" # loop 3 times to ensure robustness.\n",
514+
" for i in range(3):\n",
515+
" \n",
516+
" # Call the LLM\n",
517+
" response = llm.invoke(validation_prompt)\n",
518+
"\n",
519+
" # Debug: print the raw LLM response\n",
520+
" # print(\"Raw LLM Response:\")\n",
521+
" # print(response)\n",
522+
"\n",
523+
" try:\n",
524+
" # Parse the response using the output parser.\n",
525+
" parsed_output = parser.parse(response.content)\n",
526+
" print(\"LLM response successfully parsed as Texdown JSON.\")\n",
527+
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
528+
" return parsed_output.model_dump()\n",
529+
" except ValidationError as ve:\n",
530+
" print(\"❌ Pydantic Validation Error:\")\n",
531+
" for error in ve.errors():\n",
532+
" print(f\" - {error['loc']}: {error['msg']}\")\n",
533+
" print(\"Raw LLM output:\")\n",
534+
" print(response.content)\n",
535+
" except Exception as e:\n",
536+
" print(\"Error parsing textdown LLM response as JSON:\")\n",
537+
" print(\"Retrying...\")\n",
538+
" time.sleep(2)"
539+
]
540+
},
541+
{
542+
"cell_type": "code",
543+
"execution_count": null,
544+
"id": "19",
545+
"metadata": {},
546+
"outputs": [],
547+
"source": [
548+
"def md_to_json(md_content: str) -> dict:\n",
549+
" \"\"\"\n",
550+
" Extracts the title and individual questions from a tutorial sheet.\n",
551+
" \n",
552+
" Args:\n",
553+
" md_content (str): The content of a set.\n",
554+
" \n",
555+
" Returns:\n",
556+
" dict: A dictionary containing the keys \"name\" and \"exercise\".\n",
557+
" If parsing fails, returns None.\n",
558+
" \"\"\"\n",
559+
" extracted_dict = extract_questions(md_content)\n",
560+
" # validated_dict = task_rules_obeyed_check(extracted_dict)\n",
561+
" content_validated_dict = content_texdown_check(extracted_dict)\n",
562+
" return content_validated_dict"
563+
]
564+
},
565+
{
566+
"cell_type": "code",
567+
"execution_count": null,
568+
"id": "20",
569+
"metadata": {},
570+
"outputs": [],
571+
"source": [
572+
"imported_tutorial = md_to_json(md_content)"
573+
]
574+
},
575+
{
576+
"cell_type": "code",
577+
"execution_count": null,
578+
"id": "21",
404579
"metadata": {},
405580
"outputs": [],
406581
"source": [
@@ -429,7 +604,7 @@
429604
},
430605
{
431606
"cell_type": "markdown",
432-
"id": "18",
607+
"id": "22",
433608
"metadata": {},
434609
"source": [
435610
"# Form JSON Schemas"
@@ -438,7 +613,7 @@
438613
{
439614
"cell_type": "code",
440615
"execution_count": null,
441-
"id": "19",
616+
"id": "23",
442617
"metadata": {},
443618
"outputs": [],
444619
"source": [
@@ -469,7 +644,7 @@
469644
{
470645
"cell_type": "code",
471646
"execution_count": null,
472-
"id": "20",
647+
"id": "24",
473648
"metadata": {},
474649
"outputs": [],
475650
"source": []

0 commit comments

Comments
 (0)