// Package eval is a reproducible, offline evaluation harness. // // It takes canonical registry entries, generates "citizen-style" variations // (typos, reordered words, abbreviations, dropped units, extra filler words) // and measures how the resolver copes. It exists so that reliability claims are // backed by numbers rather than intuition, and it is deterministic: the same // seed always produces the same cases. package eval import ( "context" "fmt" "math/rand" "sort" "strings" "unicode" "git.totmin.ru/en2zmax/addrkit/internal/domain/address" "git.totmin.ru/en2zmax/addrkit/internal/domain/match" ) // Case is one generated query plus its expectation. // // RequireExact marks variations that must still resolve to the same house unit // (case, word order, abbreviations, typos). Variations that legitimately drop // information (ordinal, house letter, fraction) do not require an exact match; // for them a different, real address is "lossy" rather than "wrong". type Case struct { Query string Expect string Category string RequireExact bool } // CategoryStat aggregates results for one variation kind. type CategoryStat struct { Total int `json:"total"` Exact int `json:"exact"` Accuracy float64 `json:"accuracy"` Partial int `json:"partial"` Ambiguous int `json:"ambiguous"` NotFound int `json:"not_found"` WrongHouse int `json:"wrong_house"` WrongStreet int `json:"wrong_street"` Lossy int `json:"lossy"` } // Report is the outcome of a run. type Report struct { Total int `json:"total"` Exact int `json:"exact"` ExactRate float64 `json:"exact_rate"` Wrong int `json:"wrong"` // safety violations on RequireExact cases WrongExamples []string `json:"wrong_examples,omitempty"` Mismatches []string `json:"mismatches,omitempty"` Categories map[string]CategoryStat `json:"categories"` } // Run evaluates all cases against the resolver. func Run(resolver *match.Resolver, cases []Case) Report { rep := Report{Categories: map[string]CategoryStat{}} for _, c := range cases { st := rep.Categories[c.Category] st.Total++ rep.Total++ res := resolver.Resolve(context.Background(), c.Query) exp := address.Parse(c.Expect) exact := false switch res.Status { case address.StatusFound: rp := address.Parse(res.Address) equivalent := rp.StreetKey == exp.StreetKey && sameUnit(rp, exp) switch { case equivalent: st.Exact++ rep.Exact++ exact = true case !c.RequireExact: // Missing information resolved to a different real address: // expected, not a safety violation. st.Lossy++ case rp.StreetKey == exp.StreetKey: st.WrongHouse++ rep.Wrong++ rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address) default: st.WrongStreet++ rep.Wrong++ rep.WrongExamples = appendExample(rep.WrongExamples, c, res.Address) } case address.StatusPartial: st.Partial++ case address.StatusAmbiguous: st.Ambiguous++ default: st.NotFound++ } if !exact { rep.Mismatches = appendMismatch(rep.Mismatches, c, res) } if st.Total > 0 { st.Accuracy = float64(st.Exact) / float64(st.Total) } rep.Categories[c.Category] = st } if rep.Total > 0 { rep.ExactRate = float64(rep.Exact) / float64(rep.Total) } return rep } // sameUnit compares house units, ignoring the structure component. func sameUnit(a, b address.ParsedAddress) bool { return a.House == b.House && a.HouseFrac == b.HouseFrac && a.HouseLetter == b.HouseLetter && a.Building == b.Building && a.BuildingFrac == b.BuildingFrac && a.BuildingLetter == b.BuildingLetter } func appendExample(examples []string, c Case, got string) []string { const maxExamples = 25 if len(examples) >= maxExamples { return examples } return append(examples, fmt.Sprintf("%s: %q -> %q (want %q)", c.Category, c.Query, got, c.Expect)) } // appendMismatch records non-exact outcomes (capped) for diagnosis. It is kept // separate from WrongExamples so safety violations stay prominent. func appendMismatch(examples []string, c Case, res match.Result) []string { const maxExamples = 60 if len(examples) >= maxExamples { return examples } got := res.Address if got == "" && len(res.Candidates) > 0 { got = res.Candidates[0].Address } return append(examples, fmt.Sprintf("%s [%s]: %q -> %q (want %q)", c.Category, res.Status, c.Query, got, c.Expect)) } // Generate samples entries and produces perturbed cases from each. func Generate(entries []address.Entry, sample int, rng *rand.Rand) []Case { cand := make([]address.Entry, 0, len(entries)) for _, e := range entries { if e.House != "" && e.NameKey != "" { cand = append(cand, e) } } rng.Shuffle(len(cand), func(i, j int) { cand[i], cand[j] = cand[j], cand[i] }) if sample > 0 && sample < len(cand) { cand = cand[:sample] } var cases []Case for _, e := range cand { cases = append(cases, perturbations(e, rng)...) } return cases } var typeWords = []string{ "улица", "переулок", "проезд", "проспект", "шоссе", "микрорайон", "бульвар", "площадь", "тупик", "спуск", "аллея", "набережная", "кольцо", "тракт", "поселок", "квартал", "территория", "дорога", } var typeAbbrev = map[string]string{ "улица": "ул.", "переулок": "пер.", "проезд": "пр.", "проспект": "пр-т", "шоссе": "ш.", "микрорайон": "мкр", "бульвар": "б-р", "площадь": "пл.", "тупик": "туп.", "набережная": "наб.", "поселок": "пос.", } // perturbations returns the variations generated for one entry. Some are only // applicable when the corresponding component exists. func perturbations(e address.Entry, rng *rand.Rand) []Case { base := strings.TrimSpace(e.Address) if base == "" { return nil } var out []Case add := func(query, category string, requireExact bool) { if query != "" && query != base { out = append(out, Case{Query: query, Expect: base, Category: category, RequireExact: requireExact}) } } // Baseline: the canonical address itself must always resolve exactly. out = append(out, Case{Query: base, Expect: base, Category: "baseline", RequireExact: true}) add(strings.ToLower(base), "case_lower", true) add(strings.ToUpper(base), "case_upper", true) add("г. Новосибирск, "+base, "city_prefix", true) add(insertDom(base, e.House), "filler_dom", true) add(corruptPunctuation(base), "punctuation", true) add(addYo(base), "yo", true) if q, ok := replaceType(base, func(t string) string { return "" }); ok { add(q, "type_removed", true) } if q, ok := replaceType(base, func(t string) string { return typeAbbrev[t] }); ok { add(q, "type_abbrev", true) } if q, ok := replaceType(base, func(t string) string { return t }); ok { add(moveTypeFront(q), "type_front", true) } if e.Building != "" { add(strings.Replace(base, "к"+e.Building, "корпус "+e.Building, 1), "corpus_synonym", true) } if hasStructure(base) { add(dropStructure(base), "structure_drop", true) } if q := mutateLongestToken(base, rng); q != "" { add(q, "typo", true) } // Information-losing variations: not required to be exact. if e.Ordinal != "" { add(dropOrdinal(base), "drop_ordinal", false) } if e.HouseLetter != "" { add(strings.Replace(base, e.House+e.HouseLetter, e.House, 1), "drop_letter", false) } if e.HouseFrac != "" { add(strings.Replace(base, e.House+"/"+e.HouseFrac, e.House, 1), "drop_fraction", false) } return out } // replaceType finds the first street-type token and rewrites it via fn. When fn // returns "" the token is removed. The bool reports whether a type was found. func replaceType(s string, fn func(t string) string) (string, bool) { tokens := strings.Fields(s) for i, tok := range tokens { low := strings.Trim(strings.ToLower(tok), ".") for _, t := range typeWords { if low == t { repl := fn(t) if repl == "" { tokens = append(tokens[:i], tokens[i+1:]...) } else { tokens[i] = repl } return strings.Join(tokens, " "), true } } } return "", false } // moveTypeFront moves the (already present) type word to the front of the // street phrase, e.g. "станционная улица 1" -> "улица станционная 1". func moveTypeFront(s string) string { tokens := strings.Fields(s) for i, tok := range tokens { low := strings.Trim(strings.ToLower(tok), ".") for _, t := range typeWords { if low == t { // Build a fresh slice: appending to tokens[:i] would alias // and overwrite tokens[i] before it is captured. rest := make([]string, 0, len(tokens)-1) rest = append(rest, tokens[:i]...) rest = append(rest, tokens[i+1:]...) return strings.Join(append([]string{tok}, rest...), " ") } } } return s } func insertDom(s, house string) string { if house == "" { return "" } idx := strings.LastIndex(s, " "+house) if idx < 0 { return "" } return s[:idx] + " дом" + s[idx:] } func corruptPunctuation(s string) string { idx := strings.Index(s, " ") if idx < 0 { return "" } return s[:idx] + ", " + s[idx+1:] } func addYo(s string) string { // Must operate on runes: a byte-index splice would cut a 2-byte UTF-8 // Cyrillic letter in half. r := []rune(s) for i, ch := range r { if ch == 'е' { r[i] = 'ё' return string(r) } } return "" } func hasStructure(s string) bool { return strings.Contains(s, " стр") || strings.Contains(s, " строение") || strings.Contains(s, " сооружение") } func dropStructure(s string) string { for _, marker := range []string{" стр.", " строение", " сооружение", " стр"} { if idx := strings.Index(s, marker); idx >= 0 { return strings.TrimSpace(s[:idx]) } } return s } func dropOrdinal(s string) string { tokens := strings.Fields(s) if len(tokens) == 0 { return "" } first := tokens[0] if strings.Contains(first, "-") || isOrdinalWord(first) { return strings.Join(tokens[1:], " ") } return "" } func isOrdinalWord(s string) bool { switch strings.ToLower(s) { case "первый", "первая", "второй", "вторая", "третий", "третья", "четвертый", "четвертая", "пятый", "пятая", "шестой", "шестая", "седьмой", "седьмая", "восьмой", "восьмая", "девятый", "девятая", "десятый", "десятая": return true } return false } // mutateLongestToken applies a single edit (delete, swap or substitute) to the // longest purely-Cyrillic *name* token, emulating a citizen typo. Type words // and numeric/ordinal tokens are excluded: corrupting "улица" must not be // treated as a typo in the street name. func mutateLongestToken(s string, rng *rand.Rand) string { tokens := strings.Fields(s) best := -1 for i, tok := range tokens { if !isCyrillicWord(tok) || isTypeWord(tok) { continue } if best < 0 || len([]rune(tok)) > len([]rune(tokens[best])) { best = i } } if best < 0 { return "" } r := []rune(tokens[best]) if len(r) < 4 { return "" } switch rng.Intn(3) { case 0: // delete i := 1 + rng.Intn(len(r)-2) r = append(r[:i], r[i+1:]...) case 1: // swap adjacent i := 1 + rng.Intn(len(r)-2) r[i], r[i+1] = r[i+1], r[i] case 2: // substitute with a neighbouring Cyrillic letter i := 1 + rng.Intn(len(r)-2) r[i] = neighbour(r[i], rng) } tokens[best] = string(r) return strings.Join(tokens, " ") } func isTypeWord(tok string) bool { low := strings.Trim(strings.ToLower(tok), ".") for _, t := range typeWords { if low == t { return true } } return false } func isCyrillicWord(s string) bool { if s == "" { return false } for _, r := range s { if !unicode.Is(unicode.Cyrillic, r) { return false } } return true } // neighbour returns a letter adjacent to r in the Russian alphabet. func neighbour(r rune, rng *rand.Rand) rune { if rng.Intn(2) == 0 { if r > 'а' { return r - 1 } return r + 1 } if r < 'я' { return r + 1 } return r - 1 } // SortedCategories returns category names in a stable order (for reporting). func (r Report) SortedCategories() []string { names := make([]string, 0, len(r.Categories)) for name := range r.Categories { names = append(names, name) } sort.Strings(names) return names }