PromptStore for managing/experimenting with prompts in standalone files outside of main code

.gitignore

@@ -363,7 +363,7 @@ cython_debug/
 #  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
-#.idea/
+.idea/
 
 ### Python Patch ###
 # Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration

freetext/assignment_stores/JSONFileAssignmentStore.py

@@ -5,15 +5,17 @@
 import json
 import os
 import uuid
+from typing import Union
+import pathlib
 
 
 class JSONFileAssignmentStore(AssignmentStore):
     """
     A AssignmentStore that stores assignments in a JSON file.
     """
 
-    def __init__(self, filename: str):
-        self._filename = filename
+    def __init__(self, path: Union[str, pathlib.Path]):
+        self._filename = path
 
     def get_assignment(self, key: AssignmentID) -> Assignment:
         """

freetext/assignment_stores/__init__.py

@@ -1,4 +1,59 @@
 from .AssignmentStore import AssignmentStore, InMemoryAssignmentStore
 from .JSONFileAssignmentStore import JSONFileAssignmentStore
+from .DynamoAssignmentStore import DynamoAssignmentStore
+from pydantic import BaseModel
+from typing import Literal, Union
 
-__all__ = ["AssignmentStore", "InMemoryAssignmentStore", "JSONFileAssignmentStore"]
+
+class InMemoryAssignmentStoreConfig(BaseModel):
+    type: str = Literal["in_memory"]
+
+
+class JSONAssignmentStoreConfig(BaseModel):
+    type: str = Literal["json"]
+    path: str = "assignments.json"
+
+
+class DynamoAssignmentStoreConfig(BaseModel):
+    type: str = Literal["dynamo"]
+    aws_access_key_id: str
+    aws_secret_access_key: str
+    aws_region: str
+    table_name: str
+
+
+AssignmentStoreConfig = Union[
+    InMemoryAssignmentStoreConfig,
+    JSONAssignmentStoreConfig,
+    DynamoAssignmentStoreConfig,
+]
+
+
+def create_assignment_store(config: AssignmentStoreConfig) -> AssignmentStore:
+    """Factory function for creating a assignment store from a config object."""
+    if config.type == Literal["in_memory"]:
+        return InMemoryAssignmentStore()
+    elif config.type == Literal["json"]:
+        return JSONFileAssignmentStore(config.path)
+    elif config.type == Literal["dynamo"]:
+        return DynamoAssignmentStore(
+            config.aws_access_key_id,
+            config.aws_secret_access_key,
+            config.aws_region,
+            config.table_name,
+        )
+    else:
+        raise ValueError(f"Unknown assignment store type: {config.type}")
+
+
+__all__ = [
+    "AssignmentStore",
+    "AssignmentStoreConfig",
+    "InMemoryAssignmentStore",
+    "InMemoryAssignmentStoreConfig",
+    "JSONFileAssignmentStore",
+    "JSONAssignmentStoreConfig",
+    "DynamoAssignmentStore",
+    "DynamoAssignmentStoreConfig",
+    "create_assignment_store",
+]

freetext/config.example.py

@@ -1,9 +1,13 @@
-from pydantic import BaseSettings
+from pydantic import BaseSettings, Field
+from freetext.assignment_stores import AssignmentStoreConfig
+from freetext.response_stores import ResponseStoreConfig
+from freetext.prompt_stores import PromptStoreConfig, PlainTextPromptStoreConfig
 
 
 class OpenAIConfig(BaseSettings):
     token: str = "sk-###"
     organization: str = "org-###"
+    model: str = "gpt-3.5-turbo"
 
 
 class ApplicationSettings(BaseSettings):
@@ -13,11 +17,10 @@ class ApplicationSettings(BaseSettings):
     # your OpenAI API key)
     assignment_creation_secret: str = "I'm totally allowed to make a project"
 
-    # AWS credentials and table names for storing assignments and responses.
-    # If you're using local (e.g., JSON-based) stores, you can set these all to
-    # empty strings or ignore them entirely.
-    aws_access_key_id: str = "AKIA###"
-    aws_secret_access_key: str = "###"
-    aws_region: str = "us-east-1"
-    assignments_table: str = "llm4_freetext_assignments"
-    responses_table: str = "llm4_freetext_responses"
+    # To override the config for stores, replace Field(..., discriminator="type") with the config you want, e.g.:
+    # assignment_store: AssignmentStoreConfig = JSONAssignmentStoreConfig(path="assignments.json")
+    # or
+    # assignment_store: AssignmentStoreConfig = InMemoryAssignmentStoreConfig()
+    assignment_store: AssignmentStoreConfig = Field(..., discriminator="type")
+    response_store: ResponseStoreConfig = Field(..., discriminator="type")
+    prompt_store: PromptStoreConfig = PlainTextPromptStoreConfig(root="prompts/default")

freetext/feedback_providers/OpenAIFeedbackProvider.py

@@ -3,6 +3,7 @@
 
 from ..config import OpenAIConfig
 from ..feedback_providers.FeedbackProvider import FeedbackProvider
+from ..prompt_stores import PromptStore
 from ..llm4text_types import Assignment, Feedback, Submission
 
 
@@ -16,11 +17,15 @@ class OpenAIChatBasedFeedbackProvider(FeedbackProvider):
     more cost-effective API use, at the cost of a more constrained prompt.
     """
 
-    def __init__(self, config_override: Optional[OpenAIConfig] = None):
+    def __init__(
+        self, prompt_store: PromptStore, config_override: Optional[OpenAIConfig] = None
+    ):
+        self.prompts = prompt_store
         if config_override is not None:
             self.config = config_override
         else:
             self.config = OpenAIConfig()
+        guidance.llm = guidance.llms.OpenAI(**self.config.dict())
 
     async def get_feedback(
         self, submission: Submission, assignment: Assignment
@@ -36,61 +41,7 @@ async def get_feedback(
 
         # set the default language model used to execute guidance programs
         try:
-            openai_kwargs = self.config.dict()
-            guidance.llm = guidance.llms.OpenAI("gpt-3.5-turbo", **openai_kwargs)
-
-            grader = guidance.Program(
-                """
-            {{#system~}}
-            You are a helpful instructor, who knows that students need precise and terse feedback. Students are most motivated if you are engaging and remain positive, but it is more important to be honest and accurate than cheerful.
-            {{~/system}}
-
-            {{#user~}}
-            The student has been given the following prompt by the instructor:
-
-            ----
-            {{prompt}}
-            ----
-
-            The secret, grader-only criteria for grading are:
-            ----
-            {{criteria}}
-            ----
-
-            Please give your OWN answer to the prompt:
-
-            {{~/user}}
-
-            {{#assistant~}}
-            {{gen '_machine_answer'}}
-            {{~/assistant}}
-
-            {{#user~}}
-            The complete student response is as follows:
-            ----
-
-            {{response}}
-
-            ----
-
-            Thinking about the differences between your answer and the student's, provide your feedback to the student as a bulleted list indicating both what the student got right and what they got wrong. Give details about what they are missing or misunderstood, and mention points they overlooked, if any.
-
-            Do not instruct the student to review the criteria, as this is not provided to the student. Write to the student using "you" in the second person. The student will not see your answer to the prompt, so do not refer to it.
-
-            Be particularly mindful of scientific rigor issues including confusing correlation with causation, biases, and logical fallacies. You must also correct factual errors using your extensive domain knowledge, even if the errors are subtle or minor.
-
-            Do not say "Keep up the good work" or other encouragement "fluff." Write only the response to the student; do not write any other text.
-
-            {{audience_caveat}}
-
-            {{fact_check_caveat}}
-            {{~/user}}
-
-            {{#assistant~}}
-            {{gen 'feedback'}}
-            {{~/assistant}}
-            """
-            )
+            grader = guidance.Program(self.prompts.get_prompt("grader.feedback"))
 
             response = submission.submission_string
             feedback = grader(
@@ -99,8 +50,6 @@ async def get_feedback(
                 criteria="\n".join(
                     [f"     * {f}" for f in assignment.feedback_requirements]
                 ),
-                audience_caveat="",  # You should provide feedback keeping in mind that the student is a Graduate Student and should be graded accordingly.
-                fact_check_caveat="You should also fact-check the student's response. If the student's response is factually incorrect, you should provide feedback on the incorrect statements.",
             )
 
             return [
@@ -128,48 +77,7 @@ async def suggest_criteria(self, assignment: Assignment) -> List[str]:
 
         """
         try:
-            openai_kwargs = self.config.dict()
-            guidance.llm = guidance.llms.OpenAI("gpt-3.5-turbo", **openai_kwargs)
-
-            grader = guidance.Program(
-                """
-            {{#system~}}
-            You are a helpful instructor, who knows that students need precise and terse feedback.
-            {{~/system}}
-
-            {{#user~}}
-            The student has been given the following prompt by the instructor:
-
-            ----
-            {{prompt}}
-            ----
-
-            The secret, grader-only criteria for grading are:
-            ----
-            {{criteria}}
-            ----
-
-            Please give your OWN answer to the prompt:
-
-            {{~/user}}
-
-            {{#assistant~}}
-            {{gen '_machine_answer'}}
-            {{~/assistant}}
-
-            {{#user~}}
-            Thinking about the important points that must be addressed in this question, provide a bulleted list of criteria that should be used to grade the student's response. These criteria should be specific and precise, and should be able to be applied to the student's response to determine a grade. You may include the criteria that were provided to the student if you agree with them, or you may modify them or replace them entirely.
-
-            In general, you should provide 3-5 criteria. You can provide fewer if you think that is appropriate.
-
-            {{audience_caveat}}
-            {{~/user}}
-
-            {{#assistant~}}
-            {{gen 'criteria'}}
-            {{~/assistant}}
-            """
-            )
+            grader = guidance.Program(self.prompts.get_prompt("grader.draft_criteria"))
 
             response = assignment.student_prompt
             criteria = grader(
@@ -204,30 +112,8 @@ async def suggest_question(self, assignment: Assignment) -> str:
 
         """
         try:
-            openai_kwargs = self.config.dict()
-            guidance.llm = guidance.llms.OpenAI("gpt-3.5-turbo", **openai_kwargs)
-
             draft_response = guidance.Program(
-                """
-            {{#system~}}
-            You are a knowledgeable assistant who is working to develop a course.
-            {{~/system}}
-
-            {{#user~}}
-            You must answer the following question to the best of your ability.
-
-            ----
-            {{prompt}}
-            ----
-
-            Please give your OWN answer to this question:
-
-            {{~/user}}
-
-            {{#assistant~}}
-            {{gen '_machine_answer'}}
-            {{~/assistant}}
-            """
+                self.prompts.get_prompt("grader.draft_response")
             )
             criteria = draft_response(prompt=assignment.student_prompt)
 
@@ -240,32 +126,7 @@ async def suggest_question(self, assignment: Assignment) -> str:
             )
 
             question_improver = guidance.Program(
-                """
-            {{#system~}}
-            You are a knowledgeable instructor who is working to develop a course.
-            {{~/system}}
-
-            {{#user~}}
-            A student has been given the following prompt by the instructor:
-
-            ----
-            {{prompt}}
-            ----
-
-            The student has received the following feedback from the grader:
-
-            ----
-            {{feedback}}
-            ----
-
-            You are concerned that the student may have been confused by the question. You want to improve the question so that students are less likely to be confused. You should not change the meaning of the question, but you may clarify the question so that the requirements of the grader are more clear. Do not explicitly refer to the feedback in your question. Your question should take the form of a question that a student would be asked.
-
-            {{~/user}}
-
-            {{#assistant~}}
-            {{gen 'improved_question'}}
-            {{~/assistant}}
-            """
+                self.prompts.get_prompt("grader.improve_question")
             )
 
             improved_question = question_improver(