Skip to content

Commit b820b4c

Browse files
HarrySu123HarrySu123
committed
added extra step to sepearet extraction of question and its parts
1 parent 55af70e commit b820b4c

File tree

1 file changed

+141
-97
lines changed

1 file changed

+141
-97
lines changed

‎conversion2025/mathpix_to_llm_to_in2lambda_to_JSON.ipynb‎

Lines changed: 141 additions & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,89 @@
294294
"id": "15",
295295
"metadata": {},
296296
"outputs": [],
297+
"source": [
298+
"#define initial question model\n",
299+
"class QuestionModel(BaseModel):\n",
300+
" # full question and full solution\n",
301+
" question_content: str = Field(..., description=\"The content of the question.\")\n",
302+
" solution_content: str = Field(..., description=\"The content of the solution.\")\n",
303+
"\n",
304+
"class AllQuestionsModel(BaseModel):\n",
305+
" name: str = Field(..., description=\"Title of the set\")\n",
306+
" year: str = Field(..., description=\"Year of the set\")\n",
307+
" questions: list[QuestionModel] = Field(..., description=\"A list of questions.\")\n",
308+
"\n",
309+
"llm_task_seperate_questions = \"\"\"\n",
310+
" Your task is to extract all the individual questions and their worked solutions from the markdown content.\n",
311+
" please follow these steps carefully:\n",
312+
" 1. you can choose the name of \"AllQuestionModel\".\n",
313+
" 2. Identify the year of the tutorial, if mentioned. Otherwise, use \"0\".\n",
314+
" 3. Every character should match the original source exactly unless you're instructed to split content into fields, without adding escapes or modifications.\n",
315+
" 4. Look through the entire markdown:\n",
316+
" - Without ignoring any mentions of images, figures, or other media.\n",
317+
" - Identify full Questions, place it into question_content\n",
318+
" - Identify the full Worked Solution for each full Question.\n",
319+
" - If the Worked Solution is not found, try to find the Answers associated with it instead.\n",
320+
" - If Worked Solution or Answers are found, place it into the solution_content. Otherwise leave as empty string, \"\".\n",
321+
" 5. Output only a valid, plain, raw JSON string matching the schema above, ready to parse immediately, with no code fence or extra text. Use plain newlines (not escaped as `\\n`).\n",
322+
" 6. The Text inside the JSON should be in Lexdown:\n",
323+
" 1. preserving all LaTeX math delimiters (`$...$` and `$$...$$`) and all formatting exactly as in the input, without paraphrasing, summarizing, or simplifying any mathematical expressions or formulas.\n",
324+
" 2. do not remove or collapse blank lines.\n",
325+
" 3. Do not escape characters like `\\n` or `\\\\`.\n",
326+
" \"\"\"\n",
327+
"\n",
328+
"def extract_questions(doc_page_content: str) -> dict:\n",
329+
" # Initialise the parser for the output.\n",
330+
" parser = PydanticOutputParser(pydantic_object=AllQuestionsModel)\n",
331+
"\n",
332+
" prompt = f\"\"\"\n",
333+
" Your task is to extract a JSON with the following structure exactly:\n",
334+
" {parser.get_format_instructions()}\n",
335+
"\n",
336+
" {llm_task_seperate_questions}\n",
337+
"\n",
338+
" Input markdown:\n",
339+
" ```\n",
340+
" {doc_page_content}\n",
341+
" ```\n",
342+
" Return the JSON now.\n",
343+
" \"\"\"\n",
344+
"\n",
345+
" # tries to call the LLM multiple times to ensure robustness.\n",
346+
" for i in range(3):\n",
347+
" \n",
348+
" # Call the LLM\n",
349+
" response = llm.invoke(prompt)\n",
350+
"\n",
351+
" # Debug: print the raw LLM response\n",
352+
" # print(\"Raw LLM Response:\")\n",
353+
" # print(response)\n",
354+
"\n",
355+
" try:\n",
356+
" # Parse the response using the output parser.\n",
357+
" parsed_output = parser.parse(response.content)\n",
358+
" print(\"LLM response successfully parsed as JSON with questions.\")\n",
359+
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
360+
" return parsed_output.model_dump()\n",
361+
" except ValidationError as ve:\n",
362+
" print(\"❌ Pydantic Validation Error:\")\n",
363+
" for error in ve.errors():\n",
364+
" print(f\" - {error['loc']}: {error['msg']}\")\n",
365+
" print(\"Raw LLM output:\")\n",
366+
" print(response.content)\n",
367+
" except Exception as e:\n",
368+
" print(\"Error parsing LLM response as JSON:\")\n",
369+
" print(\"Retrying...\")\n",
370+
" time.sleep(2)\n",
371+
" \n"
372+
]
373+
},
374+
{
375+
"cell_type": "code",
376+
"execution_count": null,
377+
"id": "16",
378+
"metadata": {},
379+
"outputs": [],
297380
"source": [
298381
"# Define the schema for the tutorial output.\n",
299382
"class Set_Question(BaseModel):\n",
@@ -307,26 +390,29 @@
307390
" year: str = Field(..., description=\"Year of the set\")\n",
308391
" questions: list[Set_Question] = Field(..., description=\"List of questions in the set\")\n",
309392
"\n",
310-
"llm_task = \"\"\"\n",
311-
"Please follow these steps carefully:\n",
312-
" 1. You can decide what to call the Set.\n",
313-
" 2. Identify the year of the tutorial, if mentioned. Otherwise, use \"0\".\n",
314-
" 3. Every character should match the original source exactly unless you're instructed to split content into fields.\n",
315-
" 4. Identify the questions in the Input markdown and add them to the \"questions\" list.\n",
316-
" 5. for each question:\n",
317-
" - Title is the only field where you are allowed to name it whatever you seem fit for the question.\n",
318-
" - Identify the content of the question, which will be always visible above the individual parts. This field uses the Milkdown editor.\n",
319-
" - Identify the parts of the question (subquestions) and their worked solutions. The parts could be obvious to find, like \"a)...\", \"b)...\", etc., or they could be implied by the question itself.\n",
320-
" - If the worked solution is not given, leave the worked solution empty.\n",
321-
" - Add the parts of the question (subquestions) and their worked solutions to the \"parts\" and \"parts_solutions\" lists, respectively.\n",
322-
" 6. Output only a valid, plain, raw JSON string matching the schema above, ready to parse immediately, with no code fence or extra text. Use plain newlines (not escaped as `\\n`).\n",
323-
" 7. The Text inside the JSON should be in Lexdown:\n",
324-
" 1. preserving all LaTeX math delimiters (`$...$` and `$$...$$`) and all formatting exactly as in the input, without paraphrasing, summarizing, or simplifying any mathematical expressions or formulas.\n",
325-
" 2. do not remove or collapse blank lines.\n",
326-
" 3. Do not escape characters like `\\n` or `\\\\`.\n",
327-
"\"\"\"\n",
328-
"\n",
329-
"def extract_questions(doc_page_content: str, extra_instruction: str = \"\") -> dict:\n",
393+
"\n",
394+
"# TODO: make parts completely seperate from stem to question\n",
395+
"# ensure no answer in question itself, only in parts_solutions.\n",
396+
"llm_task_seperate_parts = \"\"\"\n",
397+
" Your task is to seperate the questions into indicidual parts and their worked solutions.\n",
398+
" Please follow these steps carefully:\n",
399+
" 1. Every character should match the original source exactly unless you're instructed to split content into fields, without adding escapes or modifications.\n",
400+
" 2. Use the same name and year.\n",
401+
" 3. For each question in questions:\n",
402+
" - Title is the only field where you are allowed to name it whatever you seem fit for the question.\n",
403+
" - Do not neglect any images, figures, or other media mentioned in the question.\n",
404+
" - Identify the stem and parts of the question, the parts may be obvious to find, like \"a)...\", \"b)...\", etc., or they could be implied by the question itself. All question must have at least one part, if there is only one part. :\n",
405+
" 1. The stem should be placed into the \"content\" field. Text in this field should be valid in the Milkdown editor. \n",
406+
" 2. the parts of the question (subquestions) should be placed into the \"parts\" field. Text in this field should be valid under Lexdown.\n",
407+
" 3. for each part, identify the worked solution/answer and place it into the \"parts_solutions\" field, if not found, leave as empty string, \"\". Text in this field should be valid under Lexdown.\n",
408+
" 4. Output only a valid, plain, raw JSON string matching the schema above, ready to parse immediately, with no code fence or extra text. Use plain newlines (not escaped as `\\n`).\n",
409+
" 5. The Text inside the JSON should be in Lexdown:\n",
410+
" 1. preserving all LaTeX math delimiters (`$...$` and `$$...$$`) and all formatting exactly as in the input, without paraphrasing, summarizing, or simplifying any mathematical expressions or formulas.\n",
411+
" 2. Do not remove or collapse blank lines.\n",
412+
" 3. Do not escape characters like `\\n` or `\\\\`.\n",
413+
" \"\"\"\n",
414+
"\n",
415+
"def extract_parts(questions_dict: dict) -> dict:\n",
330416
" \"\"\"\n",
331417
" Extracts the title and individual questions from a tutorial sheet.\n",
332418
"\n",
@@ -361,15 +447,13 @@
361447
" prompt = f\"\"\"\n",
362448
" Input markdown:\n",
363449
" ```markdown\n",
364-
" {doc_page_content}\n",
450+
" {questions_dict}\n",
365451
" ```\n",
366452
"\n",
367453
" Your task is to extract a JSON with the following structure exactly:\n",
368454
" {parser.get_format_instructions()}\n",
369455
"\n",
370-
" {llm_task}\n",
371-
"\n",
372-
" {extra_instruction}\n",
456+
" {llm_task_seperate_parts}\n",
373457
"\n",
374458
" Return the JSON now.\n",
375459
" \"\"\"\n",
@@ -387,7 +471,7 @@
387471
" try:\n",
388472
" # Parse the response using the output parser.\n",
389473
" parsed_output = parser.parse(response.content)\n",
390-
" print(\"LLM response successfully parsed as JSON.\")\n",
474+
" print(\"LLM response successfully parsed as JSON with parts.\")\n",
391475
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
392476
" return parsed_output.model_dump()\n",
393477
" except ValidationError as ve:\n",
@@ -404,7 +488,7 @@
404488
},
405489
{
406490
"cell_type": "markdown",
407-
"id": "16",
491+
"id": "17",
408492
"metadata": {},
409493
"source": [
410494
"# LLM evaluation of the content of JSON"
@@ -413,68 +497,37 @@
413497
{
414498
"cell_type": "code",
415499
"execution_count": null,
416-
"id": "17",
500+
"id": "18",
417501
"metadata": {},
418502
"outputs": [],
419503
"source": [
420-
"def task_rules_obeyed_check(extracted_dict: dict) -> dict:\n",
421-
" \"\"\"\n",
422-
" Extracts the title and individual questions from a tutorial sheet.\n",
423-
" \n",
424-
" Args:\n",
425-
" md_content (str): The content of a set.\n",
426-
" \n",
427-
" Returns:\n",
428-
" dict: A dictionary containing the keys \"name\" and \"exercise\".\n",
429-
" If parsing fails, returns None.\n",
430-
" \"\"\"\n",
431-
" json_string = json.dumps(extracted_dict, indent=2)\n",
432-
" \n",
433-
" # prompt to let llm validate the JSON.\n",
434-
" validation_prompt = f\"\"\"\n",
435-
" You were given the following rules to follow:\n",
436-
" {llm_task}\n",
437-
"\n",
438-
" please make sure that the content of the JSON followed the rules above and return the new JSON.\n",
439-
" {json_string}\n",
504+
"llm_task_expression_check = r\"\"\"\n",
505+
" Look inside the JSON object's `questions`, specifically the `content`, `parts`, and `parts_solutions` fields. Ensure that the JSON content follows these rules:\n",
506+
" 1. No extra escaping: The JSON string must contain no literal `\\\\n`, `\\\\\\\\`, or unnecessary escape sequences unless they are explicitly present in the original input text.\n",
507+
" 2. Careful to make the distinction between inline and display math, i.e. do not mess up the use of `$` and `$$`.\n",
508+
" 3. Math delimiters: All mathematical expressions must be fully enclosed within math delimiters — use `$...$` for inline math, and `$$...$$` for display math.\n",
509+
" 4. Balanced delimiters:\n",
510+
" - All `$$` and `$` must be properly opened and closed.\n",
511+
" - No unbalanced or partial math blocks.\n",
512+
" 4. Display math formatting:\n",
513+
" - The opening `$$` must appear on a new line.\n",
514+
" - The closing `$$` must also be on its own new line.\n",
515+
" - The math content must appear immediately between them, with no extra blank lines unless they are part of the input.\n",
516+
" 5. Inline math rules:\n",
517+
" - `$...$` should not span multiple lines.\n",
518+
" - Avoid using `$$` for short inline expressions.\n",
519+
" 6. Preserve LaTeX syntax:\n",
520+
" - All LaTeX commands, backslashes (`\\`), braces (`{}`, `[]`), and special characters must be preserved exactly as in the original input.\n",
521+
" - Do not add or remove escaping.\n",
522+
" 7. Blank lines:\n",
523+
" - Preserve all blank lines inside math blocks.\n",
524+
" - Outside math, follow the structure of the original input.\n",
525+
" 8. Output format:\n",
526+
" - Output a single valid JSON string.\n",
527+
" - Do not include any extra characters, explanations, or escaped formatting outside the JSON structure.\n",
440528
" \"\"\"\n",
441529
"\n",
442-
" parser = PydanticOutputParser(pydantic_object=Set)\n",
443-
" # loop 3 times to ensure robustness.\n",
444-
" for i in range(3):\n",
445-
" \n",
446-
" # Call the LLM\n",
447-
" response = llm.invoke(validation_prompt)\n",
448-
"\n",
449-
" # Debug: print the raw LLM response\n",
450-
" # print(\"Raw LLM Response:\")\n",
451-
" # print(response)\n",
452530
"\n",
453-
" try:\n",
454-
" # Parse the response using the output parser.\n",
455-
" parsed_output = parser.parse(response.content)\n",
456-
" print(\"LLM response successfully parsed as validatedJSON.\")\n",
457-
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
458-
" return parsed_output.model_dump()\n",
459-
" except ValidationError as ve:\n",
460-
" print(\"❌ Pydantic Validation Error:\")\n",
461-
" for error in ve.errors():\n",
462-
" print(f\" - {error['loc']}: {error['msg']}\")\n",
463-
" print(\"Raw LLM output:\")\n",
464-
" print(response.content)\n",
465-
" except Exception as e:\n",
466-
" print(\"Error parsing validation LLM response as JSON:\")\n",
467-
" print(\"Retrying...\")\n",
468-
" time.sleep(2)"
469-
]
470-
},
471-
{
472-
"cell_type": "code",
473-
"execution_count": null,
474-
"id": "18",
475-
"metadata": {},
476-
"outputs": [],
477-
"source": [
478531
"def content_texdown_check(validated_dict: dict) -> dict:\n",
479532
" \"\"\"\n",
480533
" Checks if the content of the JSON is in Texdown format.\n",
@@ -490,21 +543,12 @@
490543
" \n",
491544
" # prompt to let llm validate the JSON.\n",
492545
" validation_prompt = f\"\"\"\n",
493-
" Here is a JSON:\n",
546+
" {llm_task_expression_check}\n",
547+
"\n",
548+
" Input JSON:\n",
494549
" ```json\n",
495550
" {json_string}\n",
496551
" ```\n",
497-
"\n",
498-
" look inside questions, content, parts and parts_solutions, ensure that the content of the JSON follows these rules:\n",
499-
" 1. Ensure the JSON string contains no literal \"\\n\" or \"\\\\\" characters unless explicitly part of the input text.\n",
500-
" 2. All mathematical expressions and formulas must be fully enclosed within matching math delimiters: either inline math `$...$` or display math `$$...$$`.\n",
501-
" 3. Verify all `$$` delimiters are properly opened and closed; no unbalanced or partial math blocks allowed.\n",
502-
" 4. Verify all `$` delimiters are properly opened and closed; inline math should not span multiple lines.\n",
503-
" 5. Preserve all LaTeX formatting, including backslashes and braces, exactly as in the input without adding extra escaping or modifying math commands.\n",
504-
" 6. Blank lines inside math blocks must be preserved as-is.\n",
505-
" 7. Output only a valid JSON string without any additional escaping or characters.\n",
506-
"\n",
507-
"\n",
508552
" return the JSON with the content fixed if needed.\n",
509553
" \"\"\"\n",
510554
"\n",
@@ -523,7 +567,7 @@
523567
" try:\n",
524568
" # Parse the response using the output parser.\n",
525569
" parsed_output = parser.parse(response.content)\n",
526-
" print(\"LLM response successfully parsed as Texdown JSON.\")\n",
570+
" print(\"LLM response successfully parsed as JSON with valid $$.\")\n",
527571
" # For Pydantic v2, use model_dump() to convert the model to a dictionary.\n",
528572
" return parsed_output.model_dump()\n",
529573
" except ValidationError as ve:\n",
@@ -556,8 +600,8 @@
556600
" dict: A dictionary containing the keys \"name\" and \"exercise\".\n",
557601
" If parsing fails, returns None.\n",
558602
" \"\"\"\n",
559-
" extracted_dict = extract_questions(md_content)\n",
560-
" # validated_dict = task_rules_obeyed_check(extracted_dict)\n",
603+
" questions_dict = extract_questions(md_content)\n",
604+
" extracted_dict = extract_parts(questions_dict)\n",
561605
" content_validated_dict = content_texdown_check(extracted_dict)\n",
562606
" return content_validated_dict"
563607
]

0 commit comments

Comments
 (0)