Add docs for Stateful and improve using your cat guide

Author: frankier

docs/make.jl

@@ -34,10 +34,15 @@ makedocs(;
     format = format,
     pages = [
         "Getting started" => "index.md",
-        "Creating a CAT" => "creating_a_cat.md",
-        "Using your CAT" => "using_your_cat.md",
+        "Guides" => [
+            "Creating a CAT" => "creating_a_cat.md",
+            "Using your CAT" => "using_your_cat.md",
+        ],
         (build_demos ? demopage : "Demo page skipped" => []),
-        "API reference" => "api.md",
+        "References" => [
+            "Stateful interface" => "stateful.md",
+            "API reference (all docstrings)" => "api.md",
+        ],
         "Contributing" => "contributing.md",
     ],
     warnonly = [:missing_docs],

docs/src/stateful.md

@@ -0,0 +1,36 @@
+# Stateful interface
+
+```@meta
+CurrentModule = ComputerAdaptiveTesting
+```
+
+```@docs
+Stateful.Stateful
+```
+
+## Interface
+
+```@docs
+Stateful.StatefulCat
+Stateful.next_item
+Stateful.ranked_items
+Stateful.item_criteria
+Stateful.add_response!
+Stateful.rollback!
+Stateful.reset!
+Stateful.set_item_bank!
+Stateful.get_responses
+Stateful.get_ability
+```
+
+## CatRules implementation
+
+There is an implementation in terms of [CatRules](@ref):
+
+```@docs
+Stateful.StatefulCatConfig
+```
+
+## Usage
+
+Just as [CatLoopConfig](@ref) can wrap [CatRules](@ref), you can also use it with any implementor of [Stateful.StatefulCat](@ref), and run using [Sim.run_cat](@ref).

docs/src/using_your_cat.md

@@ -1,24 +1,52 @@
 # Using your CAT
 
+```@meta
+CurrentModule = ComputerAdaptiveTesting
+```
+
 Now you've created your cat according to [Creating a CAT](@ref), there are
-a number of ways you can use it. This section covers a few. See also the [Examples](@ref demo-page).
+a number of ways you can use it.
+This section covers a few.
+See also the [Examples](@ref demo-page).
+
+When you've set up your CAT using [CatRules](@ref), you can wrap it in a [CatLoopConfig](@ref) and run it with [run_cat](@ref).
+
+```@docs; canonical=false
+CatLoopConfig
+run_cat
+```
 
 ## Simulating CATs
 
-TODO
+You might like to use a response memory or simulated responses to simulate your CAT using `auto_responder`:
 
-## Integrating into a user-facing applications
+```@docs; canonical=false
+Sim.auto_responder
+```
 
-TODO
+In case your data is different you might like to modify the implementation:
 
-## Evaluating CATs
+```@example
+function auto_responder(responses)
+    function (index, label_)
+        responses[index]
+    end
+end
+```
 
-TODO
+## Integrating into a user-facing applications
 
-## Visualising CATs
+A simple interactive implementation of `get_response(...)` is `prompt_response`:
 
-TODO
+```@docs; canonical=false
+Sim.prompt_response
+```
 
-## Constructing decision trees from CATs
+For other types of interactivity you can modify the implementation:
 
-TODO
+```@example
+function prompt_response(index_, label)
+    println("Response for $label > ")
+    parse(Int8, readline())
+end
+```

src/CatConfig.jl

@@ -2,7 +2,7 @@ module CatConfig
 
 export CatRules, CatLoopConfig
 
-using DocStringExtensions: FUNCTIONNAME, TYPEDEF, TYPEDFIELDS
+using DocStringExtensions
 using PsychometricsBazaarBase.Parameters
 
 using ..Aggregators: AbilityEstimator, AbilityTracker, ConsAbilityTracker,
@@ -114,20 +114,30 @@ function collect_trackers(next_item_rule::NextItemRule, ability_tracker::Ability
 end
 
 """
+```julia
+struct $(FUNCTIONNAME)
+$(FUNCTIONNAME)(; rules=..., get_response=..., new_response_callback=...)
+```
+$(TYPEDFIELDS)
+
 Configuration for a simulatable CAT.
 """
 @with_kw struct CatLoopConfig{CatEngineT} <: CatConfigBase
     """
-    The CAT configuration.
+    An object which implements the CAT engine.
+    Implementations exist for:
+      * [CatRules](@ref)
+      * [Stateful.StatefulCat](@ref ComputerAdaptiveTesting.Stateful.StatefulCat)
     """
     rules::CatEngineT # e.g. CatRules
     """
-    The function (index, label) -> Int8 which obtains the testee's response for
+    The function `(index, label) -> Int8`` which obtains the testee's response for
     a given question, e.g. by prompting or simulation from data.
     """
     get_response::Any
     """
-    A callback called each time there is a new responses
+    A callback called each time there is a new responses.
+    If provided, it is passed `(responses::TrackedResponses, terminating)`.
     """
     new_response_callback = nothing
 end

src/Sim.jl

@@ -1,5 +1,6 @@
 module Sim
 
+using DocStringExtensions
 using StatsBase
 using FittedItemBanks: AbstractItemBank, ResponseType
 using ..Responses
@@ -10,14 +11,18 @@ using ..NextItemRules: compute_criteria, best_item
 export run_cat, prompt_response, auto_responder
 
 """
-This response callback simply prompts 
+$(TYPEDSIGNATURES)
+
+This response callback simply prompts the user for the response using the console
 """
 function prompt_response(index_, label)
     println("Response for $label > ")
     parse(Int8, readline())
 end
 
 """
+$(TYPEDSIGNATURES)
+
 This function constructs a next item function which automatically responds
 according to `responses`.
 """
@@ -39,7 +44,13 @@ function item_label(ib_labels, next_index)
 end
 
 """
-Run a given CatLoopConfig
+```julia
+$(FUNCTIONNAME)(cat_config::CatLoopConfig, item_bank::AbstractItemBank; ib_labels=nothing)
+```
+
+Run a given [CatLoopConfig](@ref) `cat_config` on the given `item_bank`.
+If `ib_labels` is not given, default labels of the form
+`<<item #\$index>>` are passed to the callback.
 """
 function run_cat(cat_config::CatLoopConfig{RulesT},
         item_bank::AbstractItemBank;

src/Stateful.jl

@@ -1,38 +1,131 @@
+"""
+This module defines the interface for a stateful CAT as well as an implementation in terms
+of [CatRules](@ref).
+The interface is meant to enable polymorphic use of different CAT implementations.
+"""
 module Stateful
 
+using DocStringExtensions
+
 using FittedItemBanks: AbstractItemBank, ResponseType
 using ..Aggregators: TrackedResponses, Aggregators
 using ..CatConfig: CatLoopConfig, CatRules
 using ..Responses: BareResponses, Response
 using ..NextItemRules: compute_criteria, best_item
+using ..Sim: Sim, item_label
 
-export StatefulCat, StatefulCatConfig, run_cat
+export StatefulCat, StatefulCatConfig
 public next_item, ranked_items, item_criteria
 public add_response!, rollback!, reset!, get_responses, get_ability
 
-## StatefulCat interface
+"""
+$(TYPEDEF)
+
+Abstract supertype for implementation of the stateful CAT interface.
+"""
 abstract type StatefulCat end
 
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat) -> IndexT
+```
+
+Returns the index of the best next item according to the CAT.
+
+Ideally `IndexT` will be an integer and the return type a 1-based index, however it
+should at least be the same type as accepted by [add_response!](@ref).
+"""
 function next_item end
 
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat) -> AbstractVector{IndexT}
+```
+
+Return a vector of indices of the sorted from best to worst item according to the CAT.
+"""
 function ranked_items end
 
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat) -> AbstractVector{CriteriaT}
+```
+
+Returns a vector of criteria values for each item in the item bank.
+
+The criteria can vary, but should attempt to interoperate with ComputerAdaptiveTesting.jl.
+"""
 function item_criteria end
 
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat, index::IndexT, response::ResponseT)
+````
+
+The exact response type `ResponseT` depends on the item bank.
+It should be chosen to interoperate with any equivalent item bank according to the
+implementation in `ComputerAdaptiveTesting.jl`.
+"""
 function add_response! end
 
-#function add_responses! end
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat)
+```
+
+Rollback the last response added with [add_response!](@ref).
 
+Some CAT implementations may not support this operation in which case they will
+throw an error.
+"""
 function rollback! end
 
+
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat)
+```
+
+Reset the CAT to its initial state, removing all responses.
+"""
 function reset! end
 
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat, item_bank::AbstractItemBank)
+```
+
+Set the current item bank of the CAT.
+This will also reset the CAT to its initial state, removing all responses.
+
+Some CAT implementations may not support this operation in which case they will
+throw an error.
+"""
+function set_item_bank! end
+
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat) -> Tuple{AbstractVector{IndexT}, AbstractVector{ResponseT}}
+```
+
+Returns a tuple of the indices and responses of the items that have been
+added to the CAT with [add_response!](@ref) so far.
+"""
 function get_responses end
 
+"""
+```julia
+$(FUNCTIONNAME)(config::StatefulCat) -> AbilityT
+```
+
+Return the current ability estimate according to the CAT.
+The type of the ability estimate `AbilityT` depends on the CAT implementation
+but should attempt to interoperate with ComputerAdaptiveTesting.jl.
+"""
 function get_ability end
 
 ## Running the CAT
-function run_cat(cat_config::CatLoopConfig{RulesT},
+function Sim.run_cat(cat_config::CatLoopConfig{RulesT},
         ib_labels = nothing) where {RulesT <: StatefulCat}
     (; stateful_cat, get_response, new_response_callback) = cat_config
     while true
@@ -59,7 +152,13 @@ end
 
 ## TODO: Materialise the cat into a decsision tree
 
-## Implementation for CatConfig
+"""
+$(TYPEDEF)
+$(TYPEDSIGNATURES)
+
+This is a the `StatefulCat` implementation in terms of `CatRules`.
+It is also the de-facto standard for the behavior of the interface.
+"""
 struct StatefulCatConfig{TrackedResponsesT <: TrackedResponses} <: StatefulCat
     rules::CatRules
     tracked_responses::Ref{TrackedResponsesT}