Answer engine: grounded, cited synthesis with verify-then-cite

answer/answer.go

@@ -0,0 +1,73 @@
+// Package answer is the AI answer engine (architecture doc 09): the
+// retrieve-rerank-synthesize-cite pipeline that turns a query into a grounded,
+// cited answer over OpenIndex's own auditable corpus. It runs as cmd/answer,
+// behind the mixer (doc 08) and over the Synthesize and Embed gRPC boundaries.
+//
+// The governing principle is the one from doc 09: only answer what you can
+// prove. Every claim in an answer ties to a span of a specific source passage,
+// and every citation is checked by entailment before the answer is shown, so a
+// model that drifts off the evidence is caught rather than trusted. Because the
+// corpus is open, a reader can follow any citation to the archived record and
+// check it, which is the point of the whole subsystem.
+//
+// The pipeline is a four-stage funnel layered on the production retrieval stack
+// (docs 05 to 08), because synthesis quality is bounded by retrieval quality:
+//
+//	Retrieve   hybrid BM25F + dense, fused, pulls 25 to 100 candidate passages
+//	Rerank     a cross-encoder filters to the top 3 to 10 (the precision gate)
+//	Construct  reranked passages assembled with source tags and ordered for the
+//	           lost-in-the-middle effect (synth.Context)
+//	Synthesize a strict answer-only-from-context prompt, then verify and cite
+//
+// Each stage sits behind a seam so the whole pipeline is testable in-process
+// without an LLM or a network: Retriever and Reranker stand in for the serving
+// tier, synth.Synthesizer for the served model, and verify.Verifier for the NLI
+// fact-checker. The router (answer/router) decides which queries reach this
+// path at all, because running a model on every query is not survivable (doc
+// 09.5).
+package answer
+
+import "openindex"
+
+// Passage is a retrieved chunk of a document, the unit the engine works in
+// rather than a whole document, because chunk granularity is what the model
+// context and the citation spans need (doc 09.1). Score is the retrieval or
+// rerank score that ordered it; Published, when set, feeds the freshness
+// weighting in synth.
+type Passage struct {
+	Doc   openindex.GlobalDocID
+	URL   string
+	Title string
+	// Source is the host or publisher the passage came from. Synthesis
+	// consolidates per source so a single site cannot outvote the corpus by
+	// repeating itself across many passages (doc 09.3).
+	Source string
+	Text   string
+	Score  openindex.Score
+	// Published is the document timestamp from the WebTable history (doc 04),
+	// zero if unknown. It is a Unix second count to keep the domain type free
+	// of a time import; synth converts it when it needs a half-life decay.
+	Published int64
+}
+
+// Citation links a span of the answer text to the passages that support it. It
+// is the in-engine form of the grounding contract; ground.Support is the wire
+// form that ships to the client. ChunkIndices index into the passage slice the
+// answer was built from.
+type Citation struct {
+	Start        int // byte offset into Answer.Text, inclusive
+	End          int // byte offset into Answer.Text, exclusive
+	ChunkIndices []int
+}
+
+// Answer is the engine's output: the synthesized text, the passages it was
+// grounded in, and the verified citations tying spans of the text to those
+// passages. Verified is false when at least one claim failed entailment and was
+// dropped or hedged rather than shown as confident, so a caller can surface the
+// distinction (doc 09.2).
+type Answer struct {
+	Text      string
+	Passages  []Passage
+	Citations []Citation
+	Verified  bool
+}

answer/engine/citations.go

@@ -0,0 +1,118 @@
+package engine
+
+import "openindex/answer/verify"
+
+// parseCitations strips the model's inline citation markers off the answer and
+// returns the clean text plus one claim per sentence with the chunk indices it
+// cited. The model emits markers like "[1][2]" right after a sentence's
+// terminal punctuation (optionally after a space), with 1-based numbers that
+// point at the passages it was given; parseCitations turns those into 0-based
+// indices and the byte spans of the clean sentences, which is exactly the shape
+// verify.Correct and answer/ground consume.
+//
+// It works on bytes so the spans align with answer/ground, drops the marker
+// bytes and the single space that precedes them, and keeps one separating space
+// between sentences so the clean text still reads correctly. A sentence with no
+// marker becomes a claim with no citation, which verification then reports as
+// unsupported.
+func parseCitations(raw string) (string, []verify.Claim) {
+	b := []byte(raw)
+	clean := make([]byte, 0, len(b))
+	var claims []verify.Claim
+	sentStart := 0
+	i := 0
+	for i < len(b) {
+		c := b[i]
+		if !isSentenceEnd(c) {
+			clean = append(clean, c)
+			i++
+			continue
+		}
+		// Copy the run of terminal punctuation into the clean text.
+		for i < len(b) && isSentenceEnd(b[i]) {
+			clean = append(clean, b[i])
+			i++
+		}
+		end := len(clean) // the sentence, including its punctuation, ends here
+
+		// Look past a single run of whitespace for citation markers.
+		j := i
+		for j < len(b) && isSpace(b[j]) {
+			j++
+		}
+		chunks, after, ok := parseMarkers(b, j)
+
+		span := clean[sentStart:end]
+		claims = appendClaimSpan(claims, string(span), sentStart, end, chunks)
+
+		if ok {
+			// Markers consumed; drop them and the space that preceded them, then
+			// put back a single separator if more text follows.
+			i = after
+			for i < len(b) && isSpace(b[i]) {
+				i++
+			}
+			if i < len(b) {
+				clean = append(clean, ' ')
+			}
+			sentStart = len(clean)
+			continue
+		}
+		sentStart = end
+	}
+	// A trailing fragment with no terminal punctuation is its own claim.
+	if sentStart < len(clean) {
+		claims = appendClaimSpan(claims, string(clean[sentStart:]), sentStart, len(clean), nil)
+	}
+	return string(clean), claims
+}
+
+// parseMarkers reads a run of "[n]" markers starting at pos and returns the
+// 0-based chunk indices, the offset just past the run, and whether at least one
+// marker was found.
+func parseMarkers(b []byte, pos int) ([]int, int, bool) {
+	var chunks []int
+	i := pos
+	for i < len(b) && b[i] == '[' {
+		k := i + 1
+		num := 0
+		digits := false
+		for k < len(b) && b[k] >= '0' && b[k] <= '9' {
+			num = num*10 + int(b[k]-'0')
+			k++
+			digits = true
+		}
+		if !digits || k >= len(b) || b[k] != ']' {
+			break
+		}
+		if num > 0 {
+			chunks = append(chunks, num-1)
+		}
+		i = k + 1
+	}
+	return chunks, i, len(chunks) > 0
+}
+
+// appendClaimSpan trims the span text and appends it, skipping an empty span.
+func appendClaimSpan(claims []verify.Claim, text string, start, end int, chunks []int) []verify.Claim {
+	// Trim leading and trailing space from the visible text but keep the span
+	// offsets pointing at the trimmed content.
+	for start < end && isSpaceStr(text, 0) {
+		text = text[1:]
+		start++
+	}
+	for len(text) > 0 && isSpaceStr(text, len(text)-1) {
+		text = text[:len(text)-1]
+		end--
+	}
+	if start >= end || text == "" {
+		return claims
+	}
+	return append(claims, verify.Claim{Text: text, Start: start, End: end, ChunkIndices: chunks})
+}
+
+func isSentenceEnd(c byte) bool { return c == '.' || c == '!' || c == '?' }
+func isSpace(c byte) bool       { return c == ' ' || c == '\t' || c == '\n' || c == '\r' }
+func isSpaceStr(s string, i int) bool {
+	return i >= 0 && i < len(s) && isSpace(s[i])
+}

answer/engine/engine.go

@@ -0,0 +1,149 @@
+// Package engine composes the answer pipeline end to end (architecture doc
+// 09.1): it threads a query through retrieve, rerank, construct, synthesize, and
+// verify-then-cite, over the seams the other answer packages define. It lives in
+// its own package because synth and verify both import the root answer package
+// for the domain types, so the composition that imports all of them cannot.
+//
+// The pipeline is deliberately a straight line of seams, so the production
+// engine is the same control flow with the gRPC clients dropped in: Retriever
+// and Reranker stand in for the serving tier (doc 08), synth.Synthesizer for the
+// served model, and verify.Verifier for the NLI guardrail. The router (doc 09.5)
+// decides whether a query reaches the pipeline at all; Engine.Answer returns
+// ErrSearchRoute for a query the router sends to classic search, so the caller
+// short-circuits without touching the model.
+package engine
+
+import (
+	"context"
+	"errors"
+
+	"openindex/answer"
+	"openindex/answer/ground"
+	"openindex/answer/router"
+	"openindex/answer/synth"
+	"openindex/answer/verify"
+)
+
+// ErrSearchRoute is returned by Answer when the router sends the query to the
+// classic search path. It is not a failure: it tells the caller to serve web
+// results without synthesizing an answer, which is the cheap, correct outcome
+// for a navigational query.
+var ErrSearchRoute = errors.New("answer: query routed to classic search, no synthesis")
+
+// Retriever pulls candidate passages for a query. The production implementation
+// is the hybrid BM25F-plus-dense fan-out through the serving tier (docs 07, 08);
+// a test uses an in-process stub. It returns up to n passages, the wide
+// candidate set the reranker filters down.
+type Retriever interface {
+	Retrieve(ctx context.Context, query string, n int) ([]answer.Passage, error)
+}
+
+// Reranker filters a candidate set to the top k by a cross-encoder (doc 07.1),
+// the precision gate and the single largest quality lever. The production
+// implementation calls the rerank model; a test uses a stub. It must be given a
+// pool large enough to contain the answer (the candidate-pool rule, doc 07.1),
+// which is why Retrieve pulls many more than k.
+type Reranker interface {
+	Rerank(ctx context.Context, query string, passages []answer.Passage, k int) ([]answer.Passage, error)
+}
+
+// Config holds the pipeline's tunable counts. The zero value is filled with the
+// doc 09.1 defaults: a wide candidate set, a small reranked context, and the
+// 60 percent utilization sweet spot from synth.
+type Config struct {
+	CandidatePool int     // passages to retrieve before reranking
+	ContextSize   int     // passages to keep after reranking
+	PerSource     int     // max passages one source contributes (doc 09.3)
+	TokenBudget   int     // model context window in tokens
+	Utilization   float64 // fill fraction of the budget
+	MinConfidence float32 // entailment floor for a citation to survive
+	TokenCost     func(answer.Passage) int
+}
+
+// Defaults returns a Config with the doc 09.1 / 09.3 defaults. TokenCost
+// estimates four bytes per token, a rough but stable stand-in until the real
+// tokenizer is wired with the model.
+func Defaults() Config {
+	return Config{
+		CandidatePool: 50,
+		ContextSize:   8,
+		PerSource:     3,
+		TokenBudget:   8192,
+		Utilization:   synth.DefaultUtilization,
+		MinConfidence: verify.MinConfidence,
+		TokenCost:     func(p answer.Passage) int { return len(p.Text)/4 + 1 },
+	}
+}
+
+// Engine is the assembled answer pipeline. It owns the seams and the config and
+// runs them in order. It holds no per-query state, so one Engine serves many
+// concurrent queries.
+type Engine struct {
+	Classifier  router.Classifier
+	Retriever   Retriever
+	Reranker    Reranker
+	Synthesizer synth.Synthesizer
+	Verifier    verify.Verifier
+	Config      Config
+}
+
+// Answer runs the pipeline for one query and returns the grounded, cited answer.
+// It routes first and returns ErrSearchRoute for a non-model query, then
+// retrieves a wide candidate set, reranks to the context size, consolidates and
+// orders the passages for the model, synthesizes the text, and verifies every
+// claim before citing it. An answer with an unsupported claim comes back with
+// the bad claim dropped and Verified false, never with an unsupported citation
+// shown.
+func (e Engine) Answer(ctx context.Context, query string) (answer.Answer, error) {
+	cfg := e.Config
+	if cfg.CandidatePool == 0 {
+		cfg = Defaults()
+	}
+
+	if e.Classifier != nil {
+		if d := e.Classifier.Classify(query); !router.WantsModel(d.Route) {
+			return answer.Answer{}, ErrSearchRoute
+		}
+	}
+
+	candidates, err := e.Retriever.Retrieve(ctx, query, cfg.CandidatePool)
+	if err != nil {
+		return answer.Answer{}, err
+	}
+	reranked, err := e.Reranker.Rerank(ctx, query, candidates, cfg.ContextSize)
+	if err != nil {
+		return answer.Answer{}, err
+	}
+
+	// Construct the context: collapse a source that repeats itself, order for
+	// the lost-in-the-middle effect, then trim to the utilization budget.
+	passages := synth.Consolidate(reranked, cfg.PerSource)
+	passages = synth.Order(passages)
+	passages = synth.Budget(passages, cfg.TokenCost, cfg.Utilization, cfg.TokenBudget)
+
+	raw, err := e.Synthesizer.Synthesize(ctx, query, passages)
+	if err != nil {
+		return answer.Answer{}, err
+	}
+
+	// Parse the model's inline citation markers off the text, verify each claim
+	// against the passages it cited, then re-insert markers for the survivors.
+	clean, claims := parseCitations(raw)
+	result := verify.Correct(e.Verifier, claims, passages, cfg.MinConfidence)
+	text := ground.Insert(clean, result.Supports)
+
+	citations := make([]answer.Citation, 0, len(result.Supports))
+	for _, s := range result.Supports {
+		citations = append(citations, answer.Citation{
+			Start:        s.Segment.Start,
+			End:          s.Segment.End,
+			ChunkIndices: s.Chunks,
+		})
+	}
+	return answer.Answer{
+		Text:      text,
+		Passages:  passages,
+		Citations: citations,
+		Verified:  result.Verified,
+	}, nil
+}