addrkit/internal/eval/eval.go

// Package eval is a reproducible, offline evaluation harness.
//
// It takes canonical registry entries, generates "citizen-style" variations
// (typos, reordered words, abbreviations, dropped units, extra filler words)
// and measures how the resolver copes. It exists so that reliability claims are
// backed by numbers rather than intuition, and it is deterministic: the same
// seed always produces the same cases.
package eval

import (
	"context"
	"fmt"
	"math/rand"
	"sort"
	"strings"
	"unicode"

	"git.totmin.ru/en2zmax/addrkit/internal/domain/address"
	"git.totmin.ru/en2zmax/addrkit/internal/domain/match"
)

// Case is one generated query plus its expectation.
//
// RequireExact marks variations that must still resolve to the same house unit
// (case, word order, abbreviations, typos). Variations that legitimately drop
// information (ordinal, house letter, fraction) do not require an exact match;
// for them a different, real address is "lossy" rather than "wrong".
type Case struct {
	Query        string
	Expect       string
	Category     string
	RequireExact bool
}

// CategoryStat aggregates results for one variation kind.
type CategoryStat struct {
	Total       int     `json:"total"`
	Exact       int     `json:"exact"`
	Accuracy    float64 `json:"accuracy"`
	Partial     int     `json:"partial"`
	Ambiguous   int     `json:"ambiguous"`
	NotFound    int     `json:"not_found"`
	WrongHouse  int     `json:"wrong_house"`
	WrongStreet int     `json:"wrong_street"`
	Lossy       int     `json:"lossy"`
}

// Report is the outcome of a run.
type Report struct {
	Total         int                     `json:"total"`
	Exact         int                     `json:"exact"`
	ExactRate     float64                 `json:"exact_rate"`
	Wrong         int                     `json:"wrong"` // safety violations on RequireExact cases
	WrongExamples []string                `json:"wrong_examples,omitempty"`
	Mismatches    []string                `json:"mismatches,omitempty"`
	Categories    map[string]CategoryStat `json:"categories"`
}

// Run evaluates all cases against the resolver.
func Run(resolver *match.Resolver, cases []Case) Report {
	rep := Report{Categories: map[string]CategoryStat{}}
	for _, c := range cases {
		st := rep.Categories[c.Category]
		st.Total++
		rep.Total++

		res := resolver.Resolve(context.Background(), c.Query)
		exp := address.Parse(c.Expect)

		exact := false
		switch res.Status {
		case address.StatusFound:
			rp := address.Parse(res.Address)
			equivalent := rp.StreetKey == exp.StreetKey && sameUnit(rp, exp)
			switch {
			case equivalent:
				st.Exact++
				rep.Exact++
				exact = true
			case !c.RequireExact:
				// Missing information resolved to a different real address:
				// expected, not a safety violation.
				st.Lossy++
			case rp.StreetKey == exp.StreetKey:
				st.WrongHouse++
				rep.Wrong++
				rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address)
			default:
				st.WrongStreet++
				rep.Wrong++
				rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address)
			}
		case address.StatusPartial:
			st.Partial++
		case address.StatusAmbiguous:
			st.Ambiguous++
		default:
			st.NotFound++
		}
		if !exact {
			rep.Mismatches = appendMismatch(rep.Mismatches, c, res)
		}
		if st.Total > 0 {
			st.Accuracy = float64(st.Exact) / float64(st.Total)
		}
		rep.Categories[c.Category] = st
	}
	if rep.Total > 0 {
		rep.ExactRate = float64(rep.Exact) / float64(rep.Total)
	}
	return rep
}

// sameUnit compares house units, ignoring the structure component.
func sameUnit(a, b address.ParsedAddress) bool {
	return a.House == b.House &&
		a.HouseFrac == b.HouseFrac &&
		a.HouseLetter == b.HouseLetter &&
		a.Building == b.Building &&
		a.BuildingFrac == b.BuildingFrac &&
		a.BuildingLetter == b.BuildingLetter
}

func appendExample(examples []string, c Case, got string) []string {
	const maxExamples = 25
	if len(examples) >= maxExamples {
		return examples
	}
	return append(examples, fmt.Sprintf("%s: %q -> %q (want %q)", c.Category, c.Query, got, c.Expect))
}

// appendMismatch records non-exact outcomes (capped) for diagnosis. It is kept
// separate from WrongExamples so safety violations stay prominent.
func appendMismatch(examples []string, c Case, res match.Result) []string {
	const maxExamples = 60
	if len(examples) >= maxExamples {
		return examples
	}
	got := res.Address
	if got == "" && len(res.Candidates) > 0 {
		got = res.Candidates[0].Address
	}
	return append(examples, fmt.Sprintf("%s [%s]: %q -> %q (want %q)", c.Category, res.Status, c.Query, got, c.Expect))
}

// Generate samples entries and produces perturbed cases from each.
func Generate(entries []address.Entry, sample int, rng *rand.Rand) []Case {
	cand := make([]address.Entry, 0, len(entries))
	for _, e := range entries {
		if e.House != "" && e.NameKey != "" {
			cand = append(cand, e)
		}
	}
	rng.Shuffle(len(cand), func(i, j int) { cand[i], cand[j] = cand[j], cand[i] })
	if sample > 0 && sample < len(cand) {
		cand = cand[:sample]
	}

	var cases []Case
	for _, e := range cand {
		cases = append(cases, perturbations(e, rng)...)
	}
	return cases
}

var typeWords = []string{
	"улица", "переулок", "проезд", "проспект", "шоссе", "микрорайон",
	"бульвар", "площадь", "тупик", "спуск", "аллея", "набережная",
	"кольцо", "тракт", "поселок", "квартал", "территория", "дорога",
}

var typeAbbrev = map[string]string{
	"улица": "ул.", "переулок": "пер.", "проезд": "пр.", "проспект": "пр-т",
	"шоссе": "ш.", "микрорайон": "мкр", "бульвар": "б-р", "площадь": "пл.",
	"тупик": "туп.", "набережная": "наб.", "поселок": "пос.",
}

// perturbations returns the variations generated for one entry. Some are only
// applicable when the corresponding component exists.
func perturbations(e address.Entry, rng *rand.Rand) []Case {
	base := strings.TrimSpace(e.Address)
	if base == "" {
		return nil
	}
	var out []Case
	add := func(query, category string, requireExact bool) {
		if query != "" && query != base {
			out = append(out, Case{Query: query, Expect: base, Category: category, RequireExact: requireExact})
		}
	}

	// Baseline: the canonical address itself must always resolve exactly.
	out = append(out, Case{Query: base, Expect: base, Category: "baseline", RequireExact: true})

	add(strings.ToLower(base), "case_lower", true)
	add(strings.ToUpper(base), "case_upper", true)
	add("г. Новосибирск, "+base, "city_prefix", true)
	add(insertDom(base, e.House), "filler_dom", true)
	add(corruptPunctuation(base), "punctuation", true)
	add(addYo(base), "yo", true)

	if q, ok := replaceType(base, func(t string) string { return "" }); ok {
		add(q, "type_removed", true)
	}
	if q, ok := replaceType(base, func(t string) string { return typeAbbrev[t] }); ok {
		add(q, "type_abbrev", true)
	}
	if q, ok := replaceType(base, func(t string) string { return t }); ok {
		add(moveTypeFront(q), "type_front", true)
	}
	if e.Building != "" {
		add(strings.Replace(base, "к"+e.Building, "корпус "+e.Building, 1), "corpus_synonym", true)
	}
	if hasStructure(base) {
		add(dropStructure(base), "structure_drop", true)
	}
	if q := mutateLongestToken(base, rng); q != "" {
		add(q, "typo", true)
	}

	// Information-losing variations: not required to be exact.
	if e.Ordinal != "" {
		add(dropOrdinal(base), "drop_ordinal", false)
	}
	if e.HouseLetter != "" {
		add(strings.Replace(base, e.House+e.HouseLetter, e.House, 1), "drop_letter", false)
	}
	if e.HouseFrac != "" {
		add(strings.Replace(base, e.House+"/"+e.HouseFrac, e.House, 1), "drop_fraction", false)
	}
	return out
}

// replaceType finds the first street-type token and rewrites it via fn. When fn
// returns "" the token is removed. The bool reports whether a type was found.
func replaceType(s string, fn func(t string) string) (string, bool) {
	tokens := strings.Fields(s)
	for i, tok := range tokens {
		low := strings.Trim(strings.ToLower(tok), ".")
		for _, t := range typeWords {
			if low == t {
				repl := fn(t)
				if repl == "" {
					tokens = append(tokens[:i], tokens[i+1:]...)
				} else {
					tokens[i] = repl
				}
				return strings.Join(tokens, " "), true
			}
		}
	}
	return "", false
}

// moveTypeFront moves the (already present) type word to the front of the
// street phrase, e.g. "станционная улица 1" -> "улица станционная 1".
func moveTypeFront(s string) string {
	tokens := strings.Fields(s)
	for i, tok := range tokens {
		low := strings.Trim(strings.ToLower(tok), ".")
		for _, t := range typeWords {
			if low == t {
				// Build a fresh slice: appending to tokens[:i] would alias
				// and overwrite tokens[i] before it is captured.
				rest := make([]string, 0, len(tokens)-1)
				rest = append(rest, tokens[:i]...)
				rest = append(rest, tokens[i+1:]...)
				return strings.Join(append([]string{tok}, rest...), " ")
			}
		}
	}
	return s
}

func insertDom(s, house string) string {
	if house == "" {
		return ""
	}
	idx := strings.LastIndex(s, " "+house)
	if idx < 0 {
		return ""
	}
	return s[:idx] + " дом" + s[idx:]
}

func corruptPunctuation(s string) string {
	idx := strings.Index(s, " ")
	if idx < 0 {
		return ""
	}
	return s[:idx] + ",  " + s[idx+1:]
}

func addYo(s string) string {
	// Must operate on runes: a byte-index splice would cut a 2-byte UTF-8
	// Cyrillic letter in half.
	r := []rune(s)
	for i, ch := range r {
		if ch == 'е' {
			r[i] = 'ё'
			return string(r)
		}
	}
	return ""
}

func hasStructure(s string) bool {
	return strings.Contains(s, " стр") || strings.Contains(s, " строение") || strings.Contains(s, " сооружение")
}

func dropStructure(s string) string {
	for _, marker := range []string{" стр.", " строение", " сооружение", " стр"} {
		if idx := strings.Index(s, marker); idx >= 0 {
			return strings.TrimSpace(s[:idx])
		}
	}
	return s
}

func dropOrdinal(s string) string {
	tokens := strings.Fields(s)
	if len(tokens) == 0 {
		return ""
	}
	first := tokens[0]
	if strings.Contains(first, "-") || isOrdinalWord(first) {
		return strings.Join(tokens[1:], " ")
	}
	return ""
}

func isOrdinalWord(s string) bool {
	switch strings.ToLower(s) {
	case "первый", "первая", "второй", "вторая", "третий", "третья",
		"четвертый", "четвертая", "пятый", "пятая", "шестой", "шестая",
		"седьмой", "седьмая", "восьмой", "восьмая", "девятый", "девятая",
		"десятый", "десятая":
		return true
	}
	return false
}

// mutateLongestToken applies a single edit (delete, swap or substitute) to the
// longest purely-Cyrillic *name* token, emulating a citizen typo. Type words
// and numeric/ordinal tokens are excluded: corrupting "улица" must not be
// treated as a typo in the street name.
func mutateLongestToken(s string, rng *rand.Rand) string {
	tokens := strings.Fields(s)
	best := -1
	for i, tok := range tokens {
		if !isCyrillicWord(tok) || isTypeWord(tok) {
			continue
		}
		if best < 0 || len([]rune(tok)) > len([]rune(tokens[best])) {
			best = i
		}
	}
	if best < 0 {
		return ""
	}
	r := []rune(tokens[best])
	if len(r) < 4 {
		return ""
	}
	switch rng.Intn(3) {
	case 0: // delete
		i := 1 + rng.Intn(len(r)-2)
		r = append(r[:i], r[i+1:]...)
	case 1: // swap adjacent
		i := 1 + rng.Intn(len(r)-2)
		r[i], r[i+1] = r[i+1], r[i]
	case 2: // substitute with a neighbouring Cyrillic letter
		i := 1 + rng.Intn(len(r)-2)
		r[i] = neighbour(r[i], rng)
	}
	tokens[best] = string(r)
	return strings.Join(tokens, " ")
}

func isTypeWord(tok string) bool {
	low := strings.Trim(strings.ToLower(tok), ".")
	for _, t := range typeWords {
		if low == t {
			return true
		}
	}
	return false
}

func isCyrillicWord(s string) bool {
	if s == "" {
		return false
	}
	for _, r := range s {
		if !unicode.Is(unicode.Cyrillic, r) {
			return false
		}
	}
	return true
}

// neighbour returns a letter adjacent to r in the Russian alphabet.
func neighbour(r rune, rng *rand.Rand) rune {
	if rng.Intn(2) == 0 {
		if r > 'а' {
			return r - 1
		}
		return r + 1
	}
	if r < 'я' {
		return r + 1
	}
	return r - 1
}

// SortedCategories returns category names in a stable order (for reporting).
func (r Report) SortedCategories() []string {
	names := make([]string, 0, len(r.Categories))
	for name := range r.Categories {
		names = append(names, name)
	}
	sort.Strings(names)
	return names
}