feat: wrote first draft of statistics logging

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace]
-members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-literate-haskell", "harper-typst" ]
+members = [ "harper-cli", "harper-core", "harper-ls", "harper-comments", "harper-wasm", "harper-tree-sitter", "harper-html", "harper-literate-haskell", "harper-typst" , "harper-stats"]
 resolver = "2"
 
 [profile.release]

harper-cli/Cargo.toml

@@ -10,6 +10,7 @@ repository = "https://github.com/automattic/harper"
 anyhow = "1.0.97"
 ariadne = "0.4.1"
 clap = { version = "4.5.34", features = ["derive", "string"] }
+harper-stats = { path = "../harper-stats", version = "0.27.0" }
 dirs = "6.0.0"
 harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.27.0" }
 harper-core = { path = "../harper-core", version = "0.27.0" }

harper-cli/src/main.rs

@@ -1,6 +1,8 @@
 #![doc = include_str!("../README.md")]
 
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
+use std::fs::File;
+use std::io::BufReader;
 use std::path::{Component, Path, PathBuf};
 use std::sync::Arc;
 use std::{fs, process};
@@ -17,7 +19,7 @@ use harper_core::{
     MutableDictionary, TokenKind, TokenStringExt, WordId, WordMetadata,
 };
 use harper_literate_haskell::LiterateHaskellParser;
-use hashbrown::HashMap;
+use harper_stats::Stats;
 use serde::Serialize;
 
 /// A debugging tool for the Harper grammar checker.
@@ -65,6 +67,8 @@ enum Args {
     Forms { line: String },
     /// Emit a decompressed, line-separated list of the words in Harper's dictionary.
     Words,
+    /// Summarize a lint record
+    SummarizeLintRecord { file: PathBuf },
     /// Print the default config with descriptions.
     Config,
     /// Print a list of all the words in a document, sorted by frequency.
@@ -225,6 +229,16 @@ fn main() -> anyhow::Result<()> {
 
             Ok(())
         }
+        Args::SummarizeLintRecord { file } => {
+            let file = File::open(file)?;
+            let mut reader = BufReader::new(file);
+            let stats = Stats::read(&mut reader)?;
+
+            let summary = stats.summarize();
+            println!("{summary}");
+
+            Ok(())
+        }
         Args::Forms { line } => {
             let (word, annot) = line_to_parts(&line);
 

harper-core/src/document.rs

@@ -10,7 +10,9 @@ use crate::patterns::{
 };
 use crate::punctuation::Punctuation;
 use crate::vec_ext::VecExt;
-use crate::{Dictionary, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt};
+use crate::{
+    Dictionary, FatStringToken, FatToken, FstDictionary, Lrc, Token, TokenKind, TokenStringExt,
+};
 use crate::{NumberSuffix, Span};
 
 /// A document containing some amount of lexed and parsed English text.
@@ -37,6 +39,18 @@ impl Document {
             .collect()
     }
 
+    /// Locate all the tokens that intersect a provided span and convert them to [`FatToken`]s.
+    ///
+    /// Desperately needs optimization.
+    pub fn fat_tokens_intersecting(&self, span: Span) -> Vec<FatToken> {
+        let indices = self.token_indices_intersecting(span);
+
+        indices
+            .into_iter()
+            .map(|i| self.tokens[i].to_fat(&self.source))
+            .collect()
+    }
+
     /// Lexes and parses text to produce a document using a provided language
     /// parser and dictionary.
     pub fn new(text: &str, parser: &impl Parser, dictionary: &impl Dictionary) -> Self {
@@ -245,6 +259,11 @@ impl Document {
         self.tokens().map(|token| token.to_fat(&self.source))
     }
 
+    /// Get an iterator over all the tokens contained in the document.
+    pub fn fat_string_tokens(&self) -> impl Iterator<Item = FatStringToken> + '_ {
+        self.fat_tokens().map(|t| t.into())
+    }
+
     pub fn get_span_content(&self, span: &Span) -> &[char] {
         span.get_content(&self.source)
     }

harper-core/src/fat_token.rs

@@ -1,11 +1,36 @@
 use serde::{Deserialize, Serialize};
 
-use crate::TokenKind;
+use crate::{CharStringExt, TokenKind};
 
 /// A [`Token`](crate::Token) that holds its content as a fat [`Vec<char>`] rather than as a
 /// [`Span`](crate::Span).
-#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash)]
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]
 pub struct FatToken {
     pub content: Vec<char>,
     pub kind: TokenKind,
 }
+
+impl From<FatStringToken> for FatToken {
+    fn from(value: FatStringToken) -> Self {
+        Self {
+            content: value.content.chars().collect(),
+            kind: value.kind,
+        }
+    }
+}
+
+/// Similar to a [`FatToken`], but uses a [`String`] as the underlying store.
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, PartialOrd, Hash, Eq)]
+pub struct FatStringToken {
+    pub content: String,
+    pub kind: TokenKind,
+}
+
+impl From<FatToken> for FatStringToken {
+    fn from(value: FatToken) -> Self {
+        Self {
+            content: value.content.to_string(),
+            kind: value.kind,
+        }
+    }
+}

harper-core/src/lib.rs

@@ -31,7 +31,7 @@ use std::collections::VecDeque;
 pub use char_string::{CharString, CharStringExt};
 pub use currency::Currency;
 pub use document::Document;
-pub use fat_token::FatToken;
+pub use fat_token::{FatStringToken, FatToken};
 pub use ignored_lints::IgnoredLints;
 use linting::Lint;
 pub use mask::{Mask, Masker};

harper-core/src/linting/lint_group.rs

@@ -70,7 +70,7 @@ use crate::linting::{closed_compounds, phrase_corrections};
 use crate::{CharString, Dialect, Document, TokenStringExt};
 use crate::{Dictionary, MutableDictionary};
 
-#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+#[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq, Eq)]
 #[serde(transparent)]
 pub struct LintGroupConfig {
     inner: HashMap<String, Option<bool>>,

harper-core/src/linting/lint_kind.rs

@@ -6,7 +6,7 @@ use serde::{Deserialize, Serialize};
 /// The general category a [`Lint`](super::Lint) falls into.
 /// There's no reason not to add a new item here if you are adding a new rule that doesn't fit
 /// the existing categories.
-#[derive(Debug, Clone, Copy, Serialize, Deserialize, Is, Default, Hash, PartialEq)]
+#[derive(Debug, Clone, Copy, Serialize, Deserialize, Is, Default, Hash, PartialEq, Eq)]
 pub enum LintKind {
     /// This should only be used by linters doing spellcheck on individual words.
     Spelling,
@@ -22,6 +22,21 @@ pub enum LintKind {
 }
 
 impl LintKind {
+    pub fn new_from_str(s: &str) -> Option<Self> {
+        Some(match s {
+            "Spelling" => LintKind::Spelling,
+            "Capitalization" => LintKind::Capitalization,
+            "Formatting" => LintKind::Formatting,
+            "Repetition" => LintKind::Repetition,
+            "Readability" => LintKind::Readability,
+            "Miscellaneous" => LintKind::Miscellaneous,
+            "Enhancement" => LintKind::Enhancement,
+            "Word Choice" => LintKind::WordChoice,
+            "Style" => LintKind::Style,
+            _ => return None,
+        })
+    }
+
     /// Produce a string representation, which can be used as keys in a map or CSS variables.
     pub fn to_string_key(&self) -> String {
         match self {

harper-ls/Cargo.toml

@@ -8,6 +8,7 @@ readme = "README.md"
 repository = "https://github.com/automattic/harper"
 
 [dependencies]
+harper-stats = { path = "../harper-stats", version = "0.27.0" }
 harper-literate-haskell = { path = "../harper-literate-haskell", version = "0.27.0" }
 harper-core = { path = "../harper-core", version = "0.27.0", features = ["concurrent"] }
 harper-comments = { path = "../harper-comments", version = "0.27.0" }