add shannon 1-gram letters

Author: Peter Johnson

.DS_Store

@@ -0,0 +1,15 @@
+{
+  "basic_nn": {
+    "answer": "1.0",
+    "response": "1.0",
+    "model": "basic_nn",
+    "refresh": false
+  },
+  "shannon_letters_single": {
+    "answer": "2.0",
+    "response": "3.0",
+    "model": "shannon_letters_single",
+    "uniform": false,
+    "word_count": "random"
+  }
+}

evaluation_function/.DS_Store

@@ -1,4 +1,5 @@
-import sys
+import sys, argparse, json, os
+from pathlib import Path
 
 from lf_toolkit.shared.params import Params
 
@@ -7,17 +8,33 @@
 def dev():
     """Run the evaluation function from the command line for development purposes.
 
-    Usage: python -m evaluation_function.dev <answer> <response>
+    Usage: 
+    poetry run python -m evaluation_function.dev --config configs/dev.json --case basic_nn
+
+    (Change the case as desired, and ensure the dev.json is up to date with your needs)
+
     """
-    if len(sys.argv) < 3:
-        print("Usage: python -m evaluation_function.dev <answer> <response>")
-        return
-    
-    answer = sys.argv[1]
-    response = sys.argv[2]
-    model = sys.argv[3] if len(sys.argv) > 3 else "basic_nn"
-    refresh = sys.argv[4].lower() == "true" if len(sys.argv) >= 4 else False
-    params = Params(model=model, refresh=refresh)
+
+    BASE_DIR = Path(__file__).resolve().parent
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True, help="Path to JSON config")
+    parser.add_argument("--case", help="Case inside the config file")
+    args = parser.parse_args()
+
+    config_path = (BASE_DIR / args.config).resolve()
+    with open(config_path) as f:
+        all_config = json.load(f)
+
+    if args.case not in all_config: # extract config for the relevant case
+        raise ValueError(f"Case '{args.case}' not found in {args.config}")
+
+    config = all_config[args.case]
+
+    # Separate out required fields
+    answer = config.pop("answer")
+    response = config.pop("response")
+    params = Params(**config)
 
     result = evaluation_function(answer, response, params)
 

evaluation_function/dev.json

@@ -31,7 +31,10 @@ def evaluation_function(
     to output the evaluation response.
     """
 
-    model_name = getattr(params, "model", "basic_nn")  # default
+    #model_name = getattr(params, "model", "basic_nn")  # default
+    model_name = params.get("model", "basic_nn") # default
+
+    print(params)
     try:
         model = getattr(models, model_name)   # e.g. models.basic_nn
     except AttributeError:

evaluation_function/dev.py

@@ -1,3 +1,4 @@
 from . import basic_nn
+from . import shannon_letters_single
 
-__all__ = ["basic_nn"]
+__all__ = ["basic_nn", "shannon_letters_single"]

evaluation_function/evaluation.py

@@ -0,0 +1,62 @@
+import random
+import csv
+import os
+from pathlib import Path
+from io import StringIO
+import re
+
+from lf_toolkit.evaluation import Result, Params
+
+# Setup paths for saving/loading model and data
+BASE_DIR = Path(__file__).resolve().parent
+MODEL_DIR = Path(os.environ.get("MODEL_DIR", BASE_DIR / "storage"))
+MODEL_DIR.mkdir(parents=True, exist_ok=True)
+LETTERS_PATH = MODEL_DIR / "norvig_letter_frequencies.csv"
+WORD_LENGTHS_PATH = MODEL_DIR / "norvig_word_length_frequencies.csv"
+
+# Relative Frequencies of Letters in General English Plain text From Cryptographical Mathematics, by Robert Edward Lewand
+# https://web.archive.org/web/20080708193159/http://pages.central.edu/emp/LintonT/classes/spring01/cryptography/letterfreq.html
+
+def csv_to_lists(filename: str) -> list:
+    frequencies = []
+    with open(filename, newline='') as csvfile:
+        reader = csv.reader(csvfile)
+        next(reader)  # Skip header row
+        for key,value in reader:
+            frequencies.append([key, float(value)])
+    return frequencies
+
+class FrequencyData:
+    def __init__(self, filename: str = None):
+        self.tokens = []
+        self.weights = []
+        if filename:
+            data = csv_to_lists(filename)
+            self.tokens = [row[0] for row in data]
+            self.weights = [row[1] for row in data]
+
+uniform_letters = FrequencyData()
+uniform_letters.tokens  = [chr(65 + i) for i in range(26)]  # 'A' to 'Z'
+uniform_letters.tokens.append(' ')  # Add space character   
+uniform_letters.weights = [1] * 27  # Equal weights for uniform distribution    
+letters = FrequencyData(LETTERS_PATH)
+word_lengths = FrequencyData(WORD_LENGTHS_PATH)
+
+def generate_string(uniform=False,word_count=5) -> str:
+    output=[]
+    for i in range(word_count):
+        k=int(random.choices(word_lengths.tokens,weights=word_lengths.weights,k=1)[0]) 
+        if uniform:
+            output.append(''.join(random.choices(uniform_letters.tokens, weights=uniform_letters.weights,k=k)))
+        else:   
+            output.append(''.join(random.choices(letters.tokens, weights=letters.weights,k=k)))
+    output=' '.join(output)
+    return output
+
+def run(response, answer, params: Params) -> Result:
+    is_correct = True
+    word_count = params.get("word_count", 10)
+    if word_count == "random":
+        word_count = random.randint(3,15)
+    output = generate_string(uniform=params.get("uniform", False),word_count=word_count)
+    return Result(is_correct=is_correct,feedback_items=[("general",output)])

evaluation_function/models/.DS_Store

@@ -0,0 +1,27 @@
+Letter,Percent
+E,12.49
+T,9.28
+A,8.04
+O,7.64
+I,7.57
+N,7.23
+S,6.51
+R,6.28
+H,5.05
+L,4.07
+D,3.82
+C,3.34
+U,2.73
+M,2.51
+F,2.4
+P,2.14
+G,1.87
+W,1.68
+Y,1.66
+B,1.48
+V,1.05
+K,0.54
+X,0.23
+J,0.16
+Q,0.12
+Z,0.09

evaluation_function/models/__init__.py

@@ -0,0 +1,21 @@
+WordLength,Percent
+1,2.998
+2,17.343
+3,21.093
+4,17.966
+5,14.906
+6,12.57
+7,8.498
+8,5.188
+9,3.018
+10,1.669
+11,0.848
+12,0.436
+13,0.212
+14,0.098
+15,0.046
+16,0.021
+17,0.01
+18,0.005
+19,0.002
+20,0.001