stedy.go

package stedy

import (
	"math/bits"
	"unicode"
)

// LexiconVector is a 10-dimensional emotional vector from the NRC EmoLex.
// Fields are inlined for compact struct layout — no padding between float32s.
// The 10 dimensions: Joy, Trust, Positive, Anticipation (positive set); Anger,
// Fear, Sadness, Disgust, Negative (negative set); Surprise (neither).
type LexiconVector struct {
	Joy, Trust, Positive, Anticipation      float32
	Anger, Fear, Sadness, Disgust, Negative float32
	Surprise                                float32
}

// Lexicon is the lookup interface for a 10-dimensional emotional vocabulary.
// Any source of per-word emotional vectors can be plugged in — MPH hash table,
// trie, or a custom backend. The found return distinguishes "word exists but
// zero vector" from "word not in vocabulary."
type Lexicon interface {
	Lookup(word string) (vec LexiconVector, found bool)
}

// Ingester provides runtime-polymorphic access to a generic Stedy[L].
// CLI and other callers that select a lexicon at runtime use this interface
// instead of instantiating Stedy with a concrete type at compile time.
// Each *Stedy[L] implements Ingester automatically.
//
// The single interface dispatch happens per-document (at the Ingest call
// site), not per-token — runtime overhead is negligible compared to
// the generic Lookup devirtualization inside the hot loop.
type Ingester interface {
	Ingest(text string) Sentiment
	IngestCopy(text string) Sentiment
}

// Anomaly represents a single token whose smoothed emotional field differs
// too far from its raw lexicon vector. The 10 emotional dimensions are the
// smoothed field at that token, not the raw lookup.
type Anomaly struct {
	OffsetStart uint32
	OffsetEnd   uint32

	Joy, Trust, Positive, Anticipation      float32
	Anger, Fear, Sadness, Disgust, Negative float32
	Surprise                                float32
}

// Sentiment is the result of a single Ingest or IngestCopy call.
//
// The 10 emotional dimensions (Joy through Surprise) are the smoothed field
// of the LAST processed token — i.e., the freshest emotional state. Valence
// is the scalar projection of that field (positive sum minus negative sum).
//
// DramaIndex quantifies emotional turbulence: totalVolatility × (1 + micro +
// macro inversions). WeightedDramaIndex normalizes by token count for
// cross-text comparison.
//
// Anomalies lists tokens where the smoothing process pulled the field away
// from the raw lexicon vector. With Ingest, this slice is valid only until
// the next Ingest call on the same engine. Use IngestCopy for persistence.
type Sentiment struct {
	DramaIndex         float32
	WeightedDramaIndex float32
	Valence            float32

	Joy, Trust, Positive, Anticipation      float32
	Anger, Fear, Sadness, Disgust, Negative float32
	Surprise                                float32

	Anomalies []Anomaly
}

type tokenData struct {
	base   LexiconVector
	field  LexiconVector
	scalar float32
}

// Stedy is the sentiment analysis engine, generic over a concrete Lexicon
// implementation. The type parameter L enables compile-time devirtualization
// of the Lookup call — the compiler inlines it directly instead of routing
// through interface dispatch.
//
// Instantiate with a concrete lexicon type:
//
//	e := stedy.New(enmph.NewMphLexicon(), 0.5, 0.8, 0.20)
//
// The engine reuses internal buffers across Ingest calls. After warmup with
// the expected maximum text size, all Ingest calls are zero-allocation.
type Stedy[L Lexicon] struct {
	lexicon           L
	alpha             float32
	thetaMicro        float32
	thetaDeltaSqr     float32
	weightMicro       float32
	weightMacro       float32
	tokensBuffer      []string
	capsBuffer        []float32
	tokenStartOffsets []uint32
	tokenEndOffsets   []uint32
	tokens            []tokenData
	inversionBuffer   []int
	isAnomalyBits     []uint64
	anomalyBuffer     []Anomaly
}

// New creates a Stedy[L] with the given lexicon and tuning parameters.
//
//   - alpha: exponential smoothing factor (0 = independent tokens,
//     max = full accumulation, default 0.5).
//   - thetaMicro: curvature threshold for micro-inversion detection (default 0.8).
//   - thetaDelta: anomaly distance threshold (default 0.20). Higher = fewer
//     anomalies flagged.
//   - weightMicro: weight of micro-inversions in drama index (default 1.0).
//   - weightMacro: weight of macro-inversions in drama index (default 1.0).
//
// Allocations: internal buffers are pre-allocated for 64 tokens and grow
// on demand. After warming up with the maximum expected text size, the
// engine runs zero-allocation.
func New[L Lexicon](l L, alpha, thetaMicro, thetaDelta, weightMicro, weightMacro float32) *Stedy[L] {
	const (
		minCap    = 64
		minBitCap = (minCap + 63) >> 6
	)
	return &Stedy[L]{
		lexicon:           l,
		alpha:             alpha,
		thetaMicro:        thetaMicro,
		thetaDeltaSqr:     thetaDelta * thetaDelta,
		weightMicro:       weightMicro,
		weightMacro:       weightMacro,
		tokensBuffer:      make([]string, 0, minCap),
		capsBuffer:        make([]float32, 0, minCap),
		tokenStartOffsets: make([]uint32, 0, minCap),
		tokenEndOffsets:   make([]uint32, 0, minCap),
		tokens:            make([]tokenData, 0, minCap),
		inversionBuffer:   make([]int, 0, 16),
		isAnomalyBits:     make([]uint64, 0, minBitCap),
		anomalyBuffer:     make([]Anomaly, 0, 64),
	}
}

// NewDefault creates a Stedy[L] with recommended default parameters:
// alpha=0.5, thetaMicro=0.8, thetaDelta=0.20, weightMicro=1.0, weightMacro=1.0.
// Equivalent to New(l, 0.5, 0.8, 0.20, 1.0, 1.0).
func NewDefault[L Lexicon](l L) *Stedy[L] {
	return New(l, 0.5, 0.8, 0.20, 1.0, 1.0)
}

// Lexicon returns the engine's lexicon. Since lexicons are read-only after
// construction, the returned value can be shared across multiple engines
// without reloading the lexicon data into memory.
//
//	lex := enmph.NewMphLexicon()
//	e1 := stedy.New(lex, 0.5, 0.8, 0.20, 1.0, 1.0)
//	e2 := stedy.New(e1.Lexicon(), 0.3, 0.7, 0.15, 1.0, 1.0)
func (s *Stedy[L]) Lexicon() L {
	return s.lexicon
}

// Ingest tokenizes the input text, runs PASS 1 (emotion lookup + exponential
// smoothing + anomaly detection) and PASS 2 (volatility + inversion detection),
// then returns the Sentiment.
//
// The returned Sentiment's Anomalies slice points into the engine's internal
// reuse buffer. It is valid ONLY until the next Ingest call on the same Stedy.
// For persistence across calls, use IngestCopy.
//
// Zero allocations after warmup (when text length ≤ previous maximum).
func (s *Stedy[L]) Ingest(text string) Sentiment {
	s.reset()
	n := s.parseTokens(text)
	if n == 0 {
		return Sentiment{}
	}
	s.ensureCapacity(n)
	wordsNeeded := (n + 63) >> 6
	for k := 0; k < wordsNeeded; k++ {
		s.isAnomalyBits = append(s.isAnomalyBits, 0)
	}

	// PASS 1 - raw field + anomaly detection (unrolled first token)
	// Token 0
	bv0, _ := s.lexicon.Lookup(s.tokensBuffer[0])
	m0 := s.capsBuffer[0]
	f0 := LexiconVector{
		Joy:          bv0.Joy * m0,
		Trust:        bv0.Trust * m0,
		Positive:     bv0.Positive * m0,
		Anticipation: bv0.Anticipation * m0,
		Anger:        bv0.Anger * m0,
		Fear:         bv0.Fear * m0,
		Sadness:      bv0.Sadness * m0,
		Disgust:      bv0.Disgust * m0,
		Negative:     bv0.Negative * m0,
		Surprise:     bv0.Surprise * m0,
	}
	s0 := f0.Joy + f0.Trust + f0.Positive + f0.Anticipation -
		(f0.Anger + f0.Fear + f0.Sadness + f0.Disgust + f0.Negative)
	s.tokens = append(s.tokens, tokenData{base: bv0, field: f0, scalar: s0})
	if (f0.Joy-bv0.Joy)*(f0.Joy-bv0.Joy)+(f0.Trust-bv0.Trust)*(f0.Trust-bv0.Trust)+
		(f0.Positive-bv0.Positive)*(f0.Positive-bv0.Positive)+(f0.Anticipation-bv0.Anticipation)*(f0.Anticipation-bv0.Anticipation)+
		(f0.Anger-bv0.Anger)*(f0.Anger-bv0.Anger)+(f0.Fear-bv0.Fear)*(f0.Fear-bv0.Fear)+
		(f0.Sadness-bv0.Sadness)*(f0.Sadness-bv0.Sadness)+(f0.Disgust-bv0.Disgust)*(f0.Disgust-bv0.Disgust)+
		(f0.Negative-bv0.Negative)*(f0.Negative-bv0.Negative)+(f0.Surprise-bv0.Surprise)*(f0.Surprise-bv0.Surprise) > s.thetaDeltaSqr {
		s.isAnomalyBits[0] |= 1
	}

	// Tokens 1..n-1
	// SIMD opportunity: the 10 FMA below vectorize trivially to SSE (4× float32)
	// or AVX2 (8× float32). Build tag: amd64 → field_compute_amd64.s.
	for i := 1; i < n; i++ {
		baseVec, _ := s.lexicon.Lookup(s.tokensBuffer[i])
		m := s.capsBuffer[i]
		prevField := s.tokens[i-1].field
		a := s.alpha

		var currentField LexiconVector
		currentField.Joy = (baseVec.Joy * m) + (a * prevField.Joy)
		currentField.Trust = (baseVec.Trust * m) + (a * prevField.Trust)
		currentField.Positive = (baseVec.Positive * m) + (a * prevField.Positive)
		currentField.Anticipation = (baseVec.Anticipation * m) + (a * prevField.Anticipation)
		currentField.Anger = (baseVec.Anger * m) + (a * prevField.Anger)
		currentField.Fear = (baseVec.Fear * m) + (a * prevField.Fear)
		currentField.Sadness = (baseVec.Sadness * m) + (a * prevField.Sadness)
		currentField.Disgust = (baseVec.Disgust * m) + (a * prevField.Disgust)
		currentField.Negative = (baseVec.Negative * m) + (a * prevField.Negative)
		currentField.Surprise = (baseVec.Surprise * m) + (a * prevField.Surprise)
		posScore := currentField.Joy + currentField.Trust + currentField.Positive + currentField.Anticipation
		negScore := currentField.Anger + currentField.Fear + currentField.Sadness + currentField.Disgust + currentField.Negative
		s.tokens = append(s.tokens, tokenData{base: baseVec, field: currentField, scalar: posScore - negScore})

		// anomaly detection (moved from PASS2 — avoids second pass over baseVec)
		wordIdx := i >> 6
		bitIdx := uint(i & 63)
		if (currentField.Joy-baseVec.Joy)*(currentField.Joy-baseVec.Joy)+
			(currentField.Trust-baseVec.Trust)*(currentField.Trust-baseVec.Trust)+
			(currentField.Positive-baseVec.Positive)*(currentField.Positive-baseVec.Positive)+
			(currentField.Anticipation-baseVec.Anticipation)*(currentField.Anticipation-baseVec.Anticipation)+
			(currentField.Anger-baseVec.Anger)*(currentField.Anger-baseVec.Anger)+
			(currentField.Fear-baseVec.Fear)*(currentField.Fear-baseVec.Fear)+
			(currentField.Sadness-baseVec.Sadness)*(currentField.Sadness-baseVec.Sadness)+
			(currentField.Disgust-baseVec.Disgust)*(currentField.Disgust-baseVec.Disgust)+
			(currentField.Negative-baseVec.Negative)*(currentField.Negative-baseVec.Negative)+
			(currentField.Surprise-baseVec.Surprise)*(currentField.Surprise-baseVec.Surprise) > s.thetaDeltaSqr {
			s.isAnomalyBits[wordIdx] |= (1 << bitIdx)
		}
	}

	// PASS 2 - inversions (no lexicon lookups, no sumSquares)
	var (
		totalVolatility float32
		microInversions int
		macroInversions int
		i               = 0
	)
	const horizon = 16
	// tokens[i].scalar used directly — compiler hoists the struct offset
	for i < n-1 {
		// first derivative
		totalVolatility += abs32(s.tokens[i+1].scalar - s.tokens[i].scalar)

		// second derivative
		if i > 0 && i < n-1 {
			expectedMid := (s.tokens[i-1].scalar + s.tokens[i+1].scalar) / 2.0
			curvature := abs32(s.tokens[i].scalar - expectedMid)
			if curvature > s.thetaMicro {
				microInversions++
			}
		}

		// macro search
		// !!! branchless
		high := min(i+horizon, n-1)
		if s.tokens[i].scalar*s.tokens[high].scalar < 0 {
			lowIdx, highIdx := i, high
			for highIdx-lowIdx > 1 {
				midIdx := (lowIdx + highIdx) / 2
				if s.tokens[lowIdx].scalar*s.tokens[midIdx].scalar < 0 {
					highIdx = midIdx
				} else {
					lowIdx = midIdx
				}
			}
			alreadyFound := false
			for _, idx := range s.inversionBuffer {
				if idx == highIdx {
					alreadyFound = true
					break
				}
			}
			if !alreadyFound {
				macroInversions++
				s.inversionBuffer = append(s.inversionBuffer, highIdx)
			}
		}
		i++
	}

	// 3 - collect anomalies into reuse buffer (bits.TrailingZeros64 skips zero words)
	// Lifetime: valid until next Ingest call on same Stedy. Use IngestCopy() for persistence.
	s.anomalyBuffer = s.anomalyBuffer[:0]
	for wi, word := range s.isAnomalyBits {
		for word != 0 {
			lsb := word & -word
			bi := bits.TrailingZeros64(word)
			j := wi*64 + bi
			f := s.tokens[j].field
			s.anomalyBuffer = append(s.anomalyBuffer, Anomaly{
				OffsetStart:  s.tokenStartOffsets[j],
				OffsetEnd:    s.tokenEndOffsets[j],
				Joy:          f.Joy,
				Trust:        f.Trust,
				Positive:     f.Positive,
				Anticipation: f.Anticipation,
				Anger:        f.Anger,
				Fear:         f.Fear,
				Sadness:      f.Sadness,
				Disgust:      f.Disgust,
				Negative:     f.Negative,
				Surprise:     f.Surprise,
			})
			word ^= lsb
		}
	}
	last := s.tokens[n-1]
	return Sentiment{
		DramaIndex:         totalVolatility * (1 + s.weightMicro*float32(microInversions) + s.weightMacro*float32(macroInversions)),
		WeightedDramaIndex: (totalVolatility * (1 + s.weightMicro*float32(microInversions) + s.weightMacro*float32(macroInversions))) / float32(n),
		Valence:            last.scalar,
		Joy:                last.field.Joy,
		Trust:              last.field.Trust,
		Positive:           last.field.Positive,
		Anticipation:       last.field.Anticipation,
		Anger:              last.field.Anger,
		Fear:               last.field.Fear,
		Sadness:            last.field.Sadness,
		Disgust:            last.field.Disgust,
		Negative:           last.field.Negative,
		Surprise:           last.field.Surprise,
		Anomalies:          s.anomalyBuffer,
	}
}

// IngestCopy is like Ingest but deep-copies the anomaly slice.
// The returned Sentiment is fully independent from the engine — safe to
// persist, send across goroutines, or store alongside subsequent results.
// The anomaly copy is the only allocation difference vs Ingest.
func (s *Stedy[L]) IngestCopy(text string) Sentiment {
	result := s.Ingest(text)
	if len(result.Anomalies) > 0 {
		anomalies := make([]Anomaly, len(result.Anomalies))
		copy(anomalies, result.Anomalies)
		result.Anomalies = anomalies
	}
	return result
}

func (s *Stedy[L]) parseTokens(text string) int {
	// Fast encoding detection: scan first non-ASCII byte.
	// UTF-8 Cyrillic: 0xD0/0xD1 + continuation byte (0x80-0xBF).
	// KOI8-R Cyrillic: 0xC0-0xFF as single-byte chars (no valid UTF-8 sequences).
	for i := 0; i < len(text); i++ {
		if text[i] >= 0x80 {
			if text[i] == 0xD0 || text[i] == 0xD1 {
				if i+1 < len(text) && text[i+1] >= 0x80 && text[i+1] <= 0xBF {
					return s.parseTokensCyrillic(text)
				}
				// 0xD0/0xD1 without UTF-8 continuation → KOI-8 (letters П/Я)
				return s.parseTokensKOI8(text)
			}
			// Byte in KOI-8 Cyrillic range but not valid UTF-8 multi-byte start
			if text[i] >= 0xC0 && (i+1 >= len(text) || text[i+1] < 0x80 || text[i+1] > 0xBF) {
				return s.parseTokensKOI8(text)
			}
			return s.parseTokensUnicode(text)
		}
	}
	return s.parseTokensASCII(text)
}

func isAsciiLetter(c byte) bool {
	return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')
}

func isAsciiDigit(c byte) bool {
	return c >= '0' && c <= '9'
}

// isCyrillicLetter checks if b1,b2 form a Cyrillic UTF-8 letter.
func isCyrillicLetter(b1, b2 byte) bool {
	return (b1 == 0xD0 && b2 >= 0x90 && b2 <= 0xBF) || // А-Я а-п ё
		(b1 == 0xD1 && b2 >= 0x80 && b2 <= 0x8F) || // р-я
		(b1 == 0xD1 && b2 == 0x91) // ё
}

// isCyrillicUpper checks if b1,b2 form an uppercase Cyrillic letter.
func isCyrillicUpper(b1, b2 byte) bool {
	return (b1 == 0xD0 && b2 >= 0x90 && b2 <= 0xAF) || // А-Я
		(b1 == 0xD0 && b2 == 0x81) // Ё
}

// isKOI8Letter checks if b is a KOI8-R Cyrillic letter (single byte).
func isKOI8Letter(b byte) bool {
	return (b >= 0xC0 && b <= 0xDF) || // uppercase А-Я
		(b >= 0xE0 && b <= 0xFF) || // lowercase а-я
		b == 0xA3 || b == 0xB3 // ё, Ё
}

// isKOI8Upper checks if b is an uppercase KOI8-R Cyrillic letter.
func isKOI8Upper(b byte) bool {
	return (b >= 0xC0 && b <= 0xDF) || b == 0xB3 // А-Я, Ё
}

// parseTokensKOI8 tokenizes KOI8-R encoded text.
// Cyrillic chars are single bytes: uppercase 0xC0-0xDF, lowercase 0xE0-0xFF.
func (s *Stedy[L]) parseTokensKOI8(text string) int {
	var (
		wordStart   = -1
		isAllUpper  = true
		hasLetters  = false
		letterCount = 0
		tokenCount  = 0
	)
	for idx := 0; idx < len(text); idx++ {
		b := text[idx]

		// ASCII path
		if b < 0x80 {
			if isAsciiLetter(b) || isAsciiDigit(b) {
				if wordStart == -1 {
					wordStart = idx
					isAllUpper = true
					hasLetters = false
					letterCount = 0
				}
				if isAsciiLetter(b) {
					hasLetters = true
					letterCount++
					if b >= 'a' {
						isAllUpper = false
					}
				}
				continue
			}
			if wordStart != -1 {
				s.addToken(text, wordStart, idx, hasLetters, isAllUpper, letterCount, &tokenCount)
				wordStart = -1
			}
			continue
		}

		// KOI8-R Cyrillic letter (single byte)
		if isKOI8Letter(b) {
			if wordStart == -1 {
				wordStart = idx
				isAllUpper = true
				hasLetters = false
				letterCount = 0
			}
			hasLetters = true
			letterCount++
			if !isKOI8Upper(b) {
				isAllUpper = false
			}
			continue
		}

		// KOI8-R non-letter high byte (0x80-0xBF range, punctuation/specials)
		if wordStart != -1 {
			s.addToken(text, wordStart, idx, hasLetters, isAllUpper, letterCount, &tokenCount)
			wordStart = -1
		}
	}
	if wordStart != -1 {
		s.addToken(text, wordStart, len(text), hasLetters, isAllUpper, letterCount, &tokenCount)
	}
	return tokenCount
}

func (s *Stedy[L]) parseTokensASCII(text string) int {
	var (
		wordStart   = -1
		isAllUpper  = true
		hasLetters  = false
		letterCount = 0
		tokenCount  = 0
	)
	for idx := 0; idx < len(text); idx++ {
		c := text[idx]
		if isAsciiLetter(c) || isAsciiDigit(c) {
			if wordStart == -1 {
				wordStart = idx
				isAllUpper = true
				hasLetters = false
				letterCount = 0
			}
			if isAsciiLetter(c) {
				hasLetters = true
				letterCount++
				if c >= 'a' {
					isAllUpper = false
				}
			}
			continue
		}
		if wordStart != -1 {
			s.addToken(text, wordStart, idx, hasLetters, isAllUpper, letterCount, &tokenCount)
			wordStart = -1
		}
	}
	if wordStart != -1 {
		s.addToken(text, wordStart, len(text), hasLetters, isAllUpper, letterCount, &tokenCount)
	}
	return tokenCount
}

// parseTokensCyrillic is a byte-level tokenizer for mixed ASCII+Cyrillic UTF-8 text.
// No rune decoding, no unicode table lookups — uses direct byte range checks.
// Falls back to parseTokensUnicode for non-Cyrillic non-ASCII scripts.
func (s *Stedy[L]) parseTokensCyrillic(text string) int {
	var (
		wordStart   = -1
		isAllUpper  = true
		hasLetters  = false
		letterCount = 0
		tokenCount  = 0
		i           = 0
	)
	for i < len(text) {
		b := text[i]

		// ASCII path
		if b < 0x80 {
			if isAsciiLetter(b) {
				if wordStart == -1 {
					wordStart = i
					isAllUpper = true
					hasLetters = false
					letterCount = 0
				}
				hasLetters = true
				letterCount++
				if b >= 'a' {
					isAllUpper = false
				}
				i++
				continue
			}
			if isAsciiDigit(b) {
				if wordStart == -1 {
					wordStart = i
					isAllUpper = true
					hasLetters = false
					letterCount = 0
				}
				i++
				continue
			}
			// ASCII delimiter
			if wordStart != -1 {
				s.addToken(text, wordStart, i, hasLetters, isAllUpper, letterCount, &tokenCount)
				wordStart = -1
			}
			i++
			continue
		}

		// Cyrillic letter (2-byte UTF-8 sequence)
		if (b == 0xD0 || b == 0xD1) && i+1 < len(text) {
			b2 := text[i+1]
			if isCyrillicLetter(b, b2) {
				if wordStart == -1 {
					wordStart = i
					isAllUpper = true
					hasLetters = false
					letterCount = 0
				}
				hasLetters = true
				letterCount++
				if !isCyrillicUpper(b, b2) {
					isAllUpper = false
				}
				i += 2
				continue
			}
		}

		// Non-ASCII, non-Cyrillic: treat as delimiter
		// For unsupported scripts (Chinese, Arabic, etc.), characters are
		// not recognized as letters, so they split words. This is acceptable
		// since only English and Russian lexicons are supported.
		if wordStart != -1 {
			s.addToken(text, wordStart, i, hasLetters, isAllUpper, letterCount, &tokenCount)
			wordStart = -1
		}
		i++
	}
	if wordStart != -1 {
		s.addToken(text, wordStart, len(text), hasLetters, isAllUpper, letterCount, &tokenCount)
	}
	return tokenCount
}

func (s *Stedy[L]) parseTokensUnicode(text string) int {
	var (
		wordStart   = -1
		isAllUpper  = true
		hasLetters  = false
		letterCount = 0
		tokenCount  = 0
	)
	for idx, r := range text {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			if wordStart == -1 {
				wordStart = idx
				isAllUpper = true
				hasLetters = false
				letterCount = 0
			}
			if unicode.IsLetter(r) {
				hasLetters = true
				letterCount++
				if !unicode.IsUpper(r) {
					isAllUpper = false
				}
			}
			continue
		}
		if wordStart != -1 {
			s.addToken(text, wordStart, idx, hasLetters, isAllUpper, letterCount, &tokenCount)
			wordStart = -1
		}
	}
	if wordStart != -1 {
		s.addToken(text, wordStart, len(text), hasLetters, isAllUpper, letterCount, &tokenCount)
	}
	return tokenCount
}

func (s *Stedy[L]) addToken(text string, start, end int, hasLetters, isAllUpper bool, letterCount int, tokenCount *int) {
	if !hasLetters {
		return
	}
	subWord := text[start:end]
	multiplier := float32(1.0)
	if isAllUpper && letterCount > 2 {
		const capsMultiplier = 1.75
		multiplier = capsMultiplier
	}
	if *tokenCount < len(s.tokensBuffer) {
		s.tokensBuffer[*tokenCount] = subWord
		s.capsBuffer[*tokenCount] = multiplier
		s.tokenStartOffsets[*tokenCount] = uint32(start)
		s.tokenEndOffsets[*tokenCount] = uint32(end)
	} else {
		s.tokensBuffer = append(s.tokensBuffer, subWord)
		s.capsBuffer = append(s.capsBuffer, multiplier)
		s.tokenStartOffsets = append(s.tokenStartOffsets, uint32(start))
		s.tokenEndOffsets = append(s.tokenEndOffsets, uint32(end))
	}
	*tokenCount++
}

func (s *Stedy[L]) reset() {
	s.tokensBuffer = s.tokensBuffer[:0]
	s.capsBuffer = s.capsBuffer[:0]
	s.tokenStartOffsets = s.tokenStartOffsets[:0]
	s.tokenEndOffsets = s.tokenEndOffsets[:0]
	s.tokens = s.tokens[:0]
	s.inversionBuffer = s.inversionBuffer[:0]
	s.isAnomalyBits = s.isAnomalyBits[:0]
	s.anomalyBuffer = s.anomalyBuffer[:0]
}

func (s *Stedy[L]) ensureCapacity(n int) {
	if cap(s.tokensBuffer) < n {
		s.tokensBuffer = make([]string, 0, n)
		s.capsBuffer = make([]float32, 0, n)
		s.tokenStartOffsets = make([]uint32, 0, n)
		s.tokenEndOffsets = make([]uint32, 0, n)
		s.tokens = make([]tokenData, 0, n)
	}
}

// abs32 returns absolute value of float32 without float64 conversion.
func abs32(x float32) float32 {
	if x < 0 {
		return -x
	}
	return x
}