Merge pull request #42 from JoshDoesIT/feat/meta-requirement-evaluation

fix(llm): Achieve 100% accuracy across all eval suites

Author: JoshDoesIT

Makefile

@@ -1,4 +1,4 @@
-.PHONY: setup test test-unit test-eval test-integration lint format build clean demo
+.PHONY: setup test test-unit test-eval test-eval-fast eval-run model-compare test-integration lint format build clean demo
 
 ## Setup ──────────────────────────────────────────────────────────────
 setup: ## Install all dependencies including Ollama, llama3, and ragas
@@ -16,6 +16,15 @@ test-integration: ## Run integration tests
 test-eval: ## Run evaluation tests (requires Ollama)
 	uv run pytest tests/evaluation/ -m eval -v -s
 
+test-eval-fast: ## Quick eval: run E2E scenario only (fast iteration)
+	uv run pytest tests/evaluation/test_end_to_end_scenario.py -m eval -v -s
+
+eval-run: ## Run all eval suites via the prompt regression harness
+	uv run python tests/evaluation/eval_runner.py
+
+model-compare: ## Compare models on eval suites (MODELS="llama3 qwen2.5:14b")
+	uv run python tests/evaluation/model_compare.py $(MODELS)
+
 test-all: test test-eval ## Run everything including eval tests
 
 ## Code Quality ───────────────────────────────────────────────────────

demo/output/access_control_policy_chunks.jsonl

@@ -1,38 +1,38 @@
 [
   {
-    "common_id": "CC-7130e083",
+    "common_id": "CC-395c4a7f",
     "theme": "Policy and Procedures",
     "unified_description": "Develop, document, and disseminate access control policies and procedures.",
     "source_references": [
       "AC-1"
     ]
   },
   {
-    "common_id": "CC-966a08ae",
+    "common_id": "CC-969b4780",
     "theme": "Account Management",
     "unified_description": "Define and document the types of accounts allowed and specifically prohibited for use within the system.",
     "source_references": [
       "AC-2"
     ]
   },
   {
-    "common_id": "CC-f0c1f730",
+    "common_id": "CC-4626709f",
     "theme": "Automated System Account Management",
     "unified_description": "Support the management of system accounts using automated mechanisms.",
     "source_references": [
       "AC-2(1)"
     ]
   },
   {
-    "common_id": "CC-46fe4b36",
+    "common_id": "CC-06352f3a",
     "theme": "Protection of Information at Rest",
     "unified_description": "Protect the confidentiality and integrity of information at rest.",
     "source_references": [
       "SC-28"
     ]
   },
   {
-    "common_id": "CC-fbd55581",
+    "common_id": "CC-59f0df03",
     "theme": "PCI DSS 1.1.1",
     "unified_description": "All security policies and operational procedures that are identified in Requirement 1 are: \n\u2022\t Documented.\n\u2022\t Kept up to date.\n\u2022\t In use.\n\u2022\t Known to all affected parties. All security policies and operational procedures that are identified in Requirement 3 are: \n\u2022\tDocumented. \n\u2022\tKept up to date. \n\u2022\tIn use.\n\u2022\tKnown to all affected parties. All security policies and operational procedures that are identified in Requirement 6 are:\n\u2022\t Documented.\n\u2022\t Kept up to date.\n\u2022\t In use.\n\u2022\t Known to all affected parties. All security policies and operational procedures that are identified in Requirement 8 are:\n\u2022\t Documented.\n\u2022\t Kept up to date.\n\u2022\t In use.\n\u2022\t Known to all affected parties.",
     "source_references": [
@@ -43,7 +43,7 @@
     ]
   },
   {
-    "common_id": "CC-c4cc78f7",
+    "common_id": "CC-cfe8f63e",
     "theme": "PCI DSS 1.1.2",
     "unified_description": "Roles and responsibilities for performing activities in Requirement 1 are documented, assigned, and understood. Roles and responsibilities for performing activities in Requirement 3 are documented, assigned, and understood. Roles and responsibilities for performing activities in Requirement 6 are documented, assigned, and understood. \nNew requirement - effective immediately Roles and responsibilities for performing activities in Requirement 8 are documented, assigned, and understood.",
     "source_references": [
@@ -54,143 +54,143 @@
     ]
   },
   {
-    "common_id": "CC-09946ffa",
+    "common_id": "CC-d19f7fbf",
     "theme": "PCI DSS 1.2.1",
     "unified_description": "Configuration standards for NSC rulesets are:\n\u2022\t Defined.\n\u2022\t Implemented.\n\u2022\t Maintained.",
     "source_references": [
       "1.2.1"
     ]
   },
   {
-    "common_id": "CC-c1255c81",
+    "common_id": "CC-b122e211",
     "theme": "PCI DSS 1.2.2",
     "unified_description": "All changes to network connections and to configurations of NSCs are approved and managed in accordance with the change control process defined at Requirement 6.5.1.",
     "source_references": [
       "1.2.2"
     ]
   },
   {
-    "common_id": "CC-707aa218",
+    "common_id": "CC-7218e73e",
     "theme": "PCI DSS 1.2.3",
     "unified_description": "An accurate network diagram(s) is maintained that shows all connections between the CDE and other networks, including any wireless networks.",
     "source_references": [
       "1.2.3"
     ]
   },
   {
-    "common_id": "CC-60a96355",
+    "common_id": "CC-e9422191",
     "theme": "PCI DSS 1.2.4",
     "unified_description": "An accurate data-flow diagram(s) is maintained that meets the following:  \n\u2022\t Shows all account data flows across systems and networks.\n\u2022\t Updated as needed upon changes to the environment.",
     "source_references": [
       "1.2.4"
     ]
   },
   {
-    "common_id": "CC-878aa714",
+    "common_id": "CC-04d46536",
     "theme": "PCI DSS 3.2.1",
     "unified_description": "Account data storage is kept to a minimum through implementation of data retention and disposal policies, procedures, and processes that include at least the following:\n\u2022\t Coverage for all locations of stored account data.\n\u2022\t Coverage for any sensitive authentication data (SAD) stored prior to completion of authorization. This bullet is a best practice until its effective date; refer to",
     "source_references": [
       "3.2.1"
     ]
   },
   {
-    "common_id": "CC-8f1f8c38",
+    "common_id": "CC-9e5f3e6d",
     "theme": "PCI DSS 3.3.1",
     "unified_description": "SAD is not stored after authorization, even if encrypted. All sensitive authentication data received is rendered unrecoverable upon completion of the authorization process",
     "source_references": [
       "3.3.1"
     ]
   },
   {
-    "common_id": "CC-98c5fba5",
+    "common_id": "CC-5c676cde",
     "theme": "PCI DSS 3.3.1.1",
     "unified_description": "The full contents of any track are not stored upon completion of the authorization process.",
     "source_references": [
       "3.3.1.1"
     ]
   },
   {
-    "common_id": "CC-a236aa7a",
+    "common_id": "CC-99ea31ec",
     "theme": "PCI DSS 3.3.1.2",
     "unified_description": "The card verification code is not stored upon completion of the authorization process.",
     "source_references": [
       "3.3.1.2"
     ]
   },
   {
-    "common_id": "CC-c5008b7e",
+    "common_id": "CC-3ffed50a",
     "theme": "PCI DSS 3.3.1.3",
     "unified_description": "The personal identification number (PIN) and the PIN block are not stored upon completion of the authorization process.",
     "source_references": [
       "3.3.1.3"
     ]
   },
   {
-    "common_id": "CC-260baf6e",
+    "common_id": "CC-d2380416",
     "theme": "PCI DSS 6.2.1",
     "unified_description": "Bespoke and custom software are developed securely, as follows:\n\u2022\t Based on industry standards and/or best practices for secure development. \n\u2022\t In accordance with PCI DSS (for example, secure authentication and logging). \n\u2022\t Incorporating consideration of information security issues during each stage of the software development lifecycle.",
     "source_references": [
       "6.2.1"
     ]
   },
   {
-    "common_id": "CC-2abb7ce9",
+    "common_id": "CC-c5d468fb",
     "theme": "PCI DSS 6.2.2",
     "unified_description": "Software development personnel working on bespoke and custom software are trained at least once every 12 months as follows: \n\u2022\t On software security relevant to their job function and development languages.\n\u2022\t Including secure software design and secure coding techniques.\n\u2022\t Including, if security testing tools are used, how to use the tools for detecting vulnerabilities in software.",
     "source_references": [
       "6.2.2"
     ]
   },
   {
-    "common_id": "CC-1dbc2e64",
+    "common_id": "CC-b10e7ee9",
     "theme": "PCI DSS 6.2.3",
     "unified_description": "Bespoke and custom software is reviewed prior to being released into production or to customers, to identify and correct potential coding vulnerabilities, as follows:\n\u2022\t Code reviews ensure code is developed according to secure coding guidelines.\n\u2022\t Code reviews look for both existing and emerging software vulnerabilities.\n\u2022\t Appropriate corrections are implemented prior to release.",
     "source_references": [
       "6.2.3"
     ]
   },
   {
-    "common_id": "CC-b8a73bf2",
+    "common_id": "CC-5661e363",
     "theme": "PCI DSS 8.2.1",
     "unified_description": "All users are assigned a unique ID before access to system components or cardholder data is allowed.",
     "source_references": [
       "8.2.1"
     ]
   },
   {
-    "common_id": "CC-50c00579",
+    "common_id": "CC-15a81f0a",
     "theme": "PCI DSS 8.2.2",
     "unified_description": "Group, shared, or generic IDs, or other shared authentication credentials are only used when necessary on an exception basis, and are managed as follows:\n\u2022\t ID use is prevented unless needed for an exceptional circumstance.\n\u2022\t Use is limited to the time needed for the exceptional circumstance.\n\u2022\t Business justification for use is documented.\n\u2022\t Use is explicitly approved by management. \n\u2022\t Individual user identity is confirmed before access to an account is granted.\n\u2022\t Every action taken is attributable to an individual user.",
     "source_references": [
       "8.2.2"
     ]
   },
   {
-    "common_id": "CC-6e6a288c",
+    "common_id": "CC-ca9cb7bc",
     "theme": "PCI DSS 8.2.3",
     "unified_description": "Additional requirement for service providers only: Service providers with remote access to customer premises use unique authentication factors for each customer premises.",
     "source_references": [
       "8.2.3"
     ]
   },
   {
-    "common_id": "CC-88ba173e",
+    "common_id": "CC-9f107b62",
     "theme": "PCI DSS 8.2.4",
     "unified_description": "Addition, deletion, and modification of user IDs, authentication factors, and other identifier objects are managed as follows:\n\u2022\t Authorized with the appropriate approval.\n\u2022\t Implemented with only the privileges specified on the documented approval.",
     "source_references": [
       "8.2.4"
     ]
   },
   {
-    "common_id": "CC-acafaddc",
+    "common_id": "CC-b13f0844",
     "theme": "PCI DSS 8.2.5",
     "unified_description": "Access for terminated users is immediately revoked",
     "source_references": [
       "8.2.5"
     ]
   },
   {
-    "common_id": "CC-0858093c",
+    "common_id": "CC-c7e62ebd",
     "theme": "PCI DSS 12.1.1",
     "unified_description": "An overall information security policy is: \n\u2022\t Established.\n\u2022\t Published.\n\u2022\t Maintained.\n\u2022\t Disseminated to all relevant personnel, as well as to relevant vendors and business partners. The information security policy is:\n\u2022\t Reviewed at least once every 12 months.\n\u2022\t Updated as needed to reflect changes to business objectives or risks to the environment. The security policy clearly defines information security roles and responsibilities for all personnel, and all personnel are aware and acknowledge their information security responsibilities. Responsibility for information security is formally assigned to a Chief Information Security Officer or other information security knowledgeable member of executive management.",
     "source_references": [
@@ -201,7 +201,7 @@
     ]
   },
   {
-    "common_id": "CC-fb67c0a0",
+    "common_id": "CC-7282b4ae",
     "theme": "PCI DSS 12.2.1",
     "unified_description": "Acceptable use policies for end-user technologies are documented and implemented, including:\n\u2022\t Explicit approval by authorized parties.\n\u2022\t Acceptable uses of the technology.\n\u2022\t List of products approved by the company for employee use, including hardware and software.",
     "source_references": [