Merge pull request #45 from wjakethompson/reliability-thresholds

Probability thresholds for reliability
wjakethompson · Feb 12, 2024 · 973a3b6 · 973a3b6
2 parents 44d3e23 + f2eb511
commit 973a3b6
Show file tree

Hide file tree

Showing 12 changed files with 2,501 additions and 2,723 deletions.
diff --git a/.github/workflows/R-CMD-check.yaml b/.github/workflows/R-CMD-check.yaml
@@ -17,7 +17,7 @@ jobs:
       matrix:
         config:
           - {os: macOS-latest, r: 'release', rtools: ''}
-          - {os: windows-latest, r: 'release', rtools: '42'}
+          - {os: windows-latest, r: 'release', rtools: '43'}
           - {os: ubuntu-latest, r: 'devel', rtools: '', http-user-agent: 'release'}
           - {os: ubuntu-latest, r: 'release', rtools: ''}
           - {os: ubuntu-latest, r: 'oldrel-1', rtools: ''}

diff --git a/DESCRIPTION b/DESCRIPTION
@@ -68,7 +68,7 @@ Encoding: UTF-8
 Language: en-US
 LazyData: true
 Roxygen: list(markdown = TRUE)
-RoxygenNote: 7.2.3
+RoxygenNote: 7.3.1
 Biarch: true
 SystemRequirements: GNU make
 VignetteBuilder: knitr
diff --git a/NEWS.md b/NEWS.md
@@ -1,5 +1,7 @@
 # measr (development version)
 
+* Updated reliability functionality to allow for the calculation of accuracy and consistency with different thresholds for determining attribute classifications.
+
 # measr 1.0.0
 
 ## New documentation

diff --git a/R/reliability.R b/R/reliability.R
@@ -26,6 +26,10 @@ reliability <- function(model, ...) {
 #' probability should be reported. This is the *expected a posteriori* (EAP)
 #' reliability.
 #'
+#' @param threshold For `map_reliability`, the threshold applied to the
+#'  attribute-level probabilities for determining the binary attribute
+#'  classifications.
+#'
 #' @details The pattern-level reliability (`pattern_reliability`) statistics are
 #' described in Cui et al. (2012). Attribute-level classification reliability
 #' statistics (`map_reliability`) are described in Johnson & Sinharay (2018).
@@ -66,14 +70,14 @@ reliability <- function(model, ...) {
 #' )
 #'
 #' reliability(rstn_mdm_lcdm)
-reliability.measrdcm <- function(model, ..., force = FALSE) {
+reliability.measrdcm <- function(model, ..., threshold = 0.5, force = FALSE) {
   if ((!is.null(model$reliability) && length(model$reliability) > 0) &&
       !force) {
     return(model$reliability)
   }
 
   # coerce model into a list of values required for reliability
-  obj <- reli_list(model)
+  obj <- reli_list(model, threshold = threshold)
   att_names <- model$data$qmatrix %>%
     dplyr::select(-"item_id") %>%
     colnames()

diff --git a/R/utils-reliability.R b/R/utils-reliability.R
@@ -142,7 +142,7 @@ create_reli_list <- function(n_resp, n_att, n_class, profile_vec, class_att,
   return(ret_list)
 }
 
-reli_list <- function(model) {
+reli_list <- function(model, threshold) {
   n_resp <- dplyr::n_distinct(model$data$data$resp_id)
   n_att <- ncol(model$data$qmatrix) - 1L
   n_class  <- 2 ^ n_att
@@ -160,7 +160,7 @@ reli_list <- function(model) {
   # map estimates
   binary_att <- attr_probs %>%
     dplyr::mutate(dplyr::across(dplyr::everything(),
-                                ~dplyr::case_when(.x >= 0.5 ~ 1L,
+                                ~dplyr::case_when(.x >= threshold ~ 1L,
                                                   TRUE ~ 0L))) %>%
     as.matrix() %>%
     unname() %>%

diff --git a/man/reliability.Rd b/man/reliability.Rd
diff --git a/pkgdown/extra.css b/pkgdown/extra.css
@@ -4,7 +4,7 @@
   --nav-bkg: #023047;     /* nav bar background */
   --nav-link: #8ecae6;    /* nav bar link color */
   --nav-higl: #219ebc;    /* nav active and highlight */
-  --ver-norm: #023047;    /* pkg version background - release version */
+  --ver-norm: #219ebc;    /* pkg version background - release version */
   --ver-devp: #d7263d;    /* pkg version background - dev version */
 
   --pg-header: #023047;   /* heading color & text links */
@@ -41,9 +41,12 @@
 }
 
 /* Package version */
-.badge-danger {
-  color: #fff;
-  background-color: var(--ver-devp);
+small.text-danger {
+  color: var(--ver-devp) !important;
+}
+
+small.text-muted {
+  color: var(--ver-norm) !important;
 }
 
 /* Color of links in navbar and hover links */